Felipe97/llama-cpp-compiled

1// tests the HF model resolution and the model handler assembly end-to-end on2// synthetic repo listings: a local httplib server bound to the loopback3// serves hardcoded HF API responses, so the real client, hf_cache, resolution4// and CLI parsing run against them without external network access5 6#include "arg.h"7#include "common.h"8#include "download.h"9#include "http.h"10#include "log.h"11 12#include "json.h"13 14#include <algorithm>15#include <cstdio>16#include <cstdlib>17#include <filesystem>18#include <map>19#include <thread>20#include <string>21#include <vector>22 23// the case and reordering being checked, printed with every failure24static std::string g_context;25 26// independent of NDEBUG, so the checks stay alive in Release builds27#define REQUIRE(x) do {                                                         \28    if (!(x)) {                                                                 \29        fprintf(stderr, "%s:%d: [%s] REQUIRE(%s) failed\n",                     \30                __FILE__, __LINE__, g_context.c_str(), #x);                     \31        std::abort();                                                           \32    }                                                                           \33} while (0)34 35#define REQUIRE_EQ(actual, expected) do {                                       \36    if (!((actual) == (expected))) {                                            \37        fprintf(stderr, "%s:%d: [%s] REQUIRE_EQ(%s, %s) failed\n  actual:   '%s'\n  expected: '%s'\n", \38                __FILE__, __LINE__, g_context.c_str(), #actual, #expected,      \39                std::string(actual).c_str(), std::string(expected).c_str());    \40        std::abort();                                                           \41    }                                                                           \42} while (0)43 44//45// synthetic repos keyed by repo id, served over the loopback by a real46// httplib server, so the tested code runs its own client and transport47//48 49static std::map<std::string, std::vector<std::string>> g_repos;50 51static const char * COMMIT = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";52 53// the server lives in main, so its destructor runs before the static teardown54// tears down the winsock state httplib brings in55static void serve_repos(httplib::Server & server) {56    server.Get(R"(/api/models/(.+)/refs)", [](const httplib::Request & req, httplib::Response & res) {57        if (g_repos.count(req.matches[1])) {58            res.set_content(common_json{{"branches", common_json::array({ common_json{{"name", "main"}, {"targetCommit", COMMIT}} })}}.dump(),59                            "application/json");60        } else {61            res.status = 404;62        }63    });64    server.Get(R"(/api/models/(.+)/tree/.+)", [](const httplib::Request & req, httplib::Response & res) {65        if (!g_repos.count(req.matches[1])) {66            res.status = 404;67            return;68        }69        auto files = common_json::array();70        size_t i = 0;71        for (const auto & p : g_repos[req.matches[1]]) {72            char oid[41];73            snprintf(oid, sizeof(oid), "%040lx", (unsigned long) ++i);74            files.push_back({{"type", "file"}, {"path", p}, {"size", 1}, {"oid", oid}});75        }76        res.set_content(files.dump(), "application/json");77    });78}79 80static common_params_model model_ref(const std::string & hf_repo, const std::string & hf_file = "") {81    common_params_model m;82    m.hf_repo = hf_repo;83    m.hf_file = hf_file;84    return m;85}86 87// the model cache is isolated under a temporary directory named after the88// loopback port, so concurrent runs on a shared machine keep their own, and89// the local path the handler wires for a file is snapshots/<commit>/<path>90static std::filesystem::path cache_dir;91 92static std::string cached(std::string repo_id, const std::string & path) {93    string_replace_all(repo_id, "/", "--");94    return (cache_dir / ("models--" + repo_id) / "snapshots" / COMMIT / path).string();95}96 97//98// fixtures mimicking real repo layouts99//100 101// flat layout in the style of ggml-org/gemma-4-31B-it-GGUF102static const std::vector<std::string> flat = {103    "README.md",104    "model-BF16.gguf",105    "model-Q4_K_M.gguf",106    "model-Q8_0.gguf",107    "mmproj-model-BF16.gguf",108    "mmproj-model-Q8_0.gguf",109    "mtp-model-BF16.gguf",110    "mtp-model-Q4_0.gguf",111    "mtp-model-Q8_0.gguf",112    "dflash-model-BF16.gguf",113    "dflash-model-Q8_0.gguf",114};115 116// quants in subdirectories with sharded files and root sidecars,117// in the style of stepfun-ai/Step-3.7-Flash-GGUF118static const std::vector<std::string> subdir = {119    "mmproj-model-f16.gguf",120    "model-mtp-BF16.gguf",121    "model-mtp-Q8_0.gguf",122    "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",123    "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf",124    "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf",125    "Q8_0/model-Q8_0-00001-of-00002.gguf",126    "Q8_0/model-Q8_0-00002-of-00002.gguf",127};128 129// sidecar quants exist where the full model quant does not,130// in the style of ggml-org/Qwen3.6-27B-GGUF131static const std::vector<std::string> hole = {132    "model-BF16.gguf",133    "model-Q4_K_M.gguf",134    "model-Q8_0.gguf",135    "mtp-model-BF16.gguf",136    "mtp-model-Q4_0.gguf",137    "mtp-model-Q8_0.gguf",138    "dflash-model-BF16.gguf",139    "dflash-model-Q8_0.gguf",140};141 142// unsloth-style naming with UD quants and a suffix MTP file143static const std::vector<std::string> unsloth = {144    "model-UD-Q8_K_XL.gguf",145    "mmproj-BF16.gguf",146    "model-MTP-BF16.gguf",147};148 149// bartowski-style vendor prefix and mradermacher-style dot quant150static const std::vector<std::string> vendors = {151    "TheDrummer_Model-24B-v4.1-Q8_0.gguf",152    "BlackSheep-24B.Q8_0.gguf",153};154 155// every speculative sidecar type at the same quant156static const std::vector<std::string> quad = {157    "model-Q8_0.gguf",158    "mtp-model-Q8_0.gguf",159    "dflash-model-Q8_0.gguf",160    "eagle3-model-Q8_0.gguf",161    "dspark-model-Q8_0.gguf",162};163 164static const std::vector<std::string> dflash_only = {165    "model-Q8_0.gguf",166    "dflash-model-Q8_0.gguf",167};168 169static const std::vector<std::string> eagle3_only = {170    "model-Q8_0.gguf",171    "eagle3-model-Q8_0.gguf",172};173 174// a single full quant with dspark sidecars at other quants,175// in the style of ggml-org/DeepSeek-V4-Flash-0731-GGUF176static const std::vector<std::string> spark = {177    "README.md",178    "model-MXFP4.gguf",179    "dspark-model-BF16.gguf",180    "dspark-model-MXFP4.gguf",181};182 183// dspark outranks dflash in the type auto-selection184static const std::vector<std::string> dspark_dflash = {185    "model-Q8_0.gguf",186    "dflash-model-Q8_0.gguf",187    "dspark-model-Q8_0.gguf",188};189 190//191// table-driven plan resolution through the real entry point,192// each case replayed on multiple deterministic reorderings of the listing,193// except the cases whose pick legitimately depends on the listing order194//195 196struct plan_case {197    const char * name;198    const std::vector<std::string> files;199    const char * hf_repo;200    const char * hf_file;201    bool sidecars;        // request mmproj + mtp + dflash + eagle3 + dspark202    bool order_dependent; // the expected pick depends on the listing order203    const char * primary;204    std::vector<std::string> model_files;205    const char * mmproj;206    const char * mtp;207    const char * dflash;208    const char * eagle3;209    const char * dspark;210};211 212static const plan_case plan_cases[] = {213    // exact tag picks the matching primary, sidecars follow the tag214    {"flat exact tag", flat, "test/repo:Q8_0", "", true, false,215     "model-Q8_0.gguf", {"model-Q8_0.gguf"},216     "mmproj-model-Q8_0.gguf", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "", ""},217 218    // no tag falls back to the default quant preference219    {"flat default", flat, "test/repo", "", false, false,220     "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"},221     "", "", "", "", ""},222 223    // no tag and no default match falls back to the first model in the listing224    {"unsloth fallback", unsloth, "test/repo", "", true, true,225     "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"},226     "mmproj-BF16.gguf", "", "", "", ""},227 228    // explicit hf_file picks that exact file229    {"flat hf_file", flat, "test/repo", "model-BF16.gguf", false, false,230     "model-BF16.gguf", {"model-BF16.gguf"},231     "", "", "", "", ""},232 233    // missing hf_file resolves nothing234    {"flat missing hf_file", flat, "test/repo", "nope.gguf", false, false,235     "", {},236     "", "", "", "", ""},237 238    // a sharded primary brings all its parts, a subdir primary finds the root sidecar239    {"subdir shards", subdir, "test/repo:Q3_K_M", "", true, false,240     "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",241     {"Q3_K_M/model-Q3_K_M-00001-of-00003.gguf",242      "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf",243      "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf"},244     "mmproj-model-f16.gguf", "model-mtp-Q8_0.gguf", "", "", ""},245 246    // a tag with no matching full model still resolves the requested sidecars247    {"hole tag sidecar", hole, "test/repo:Q4_0", "", true, false,248     "", {},249     "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""},250 251    // the same tag without a requested sidecar resolves nothing252    {"hole tag alone", hole, "test/repo:Q4_0", "", false, false,253     "", {},254     "", "", "", "", ""},255 256    // no tag anchors the sidecars on the primary quant257    {"hole default anchor", hole, "test/repo", "", true, false,258     "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"},259     "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", "", ""},260 261    // the mtp- keyword is case sensitive, a suffix -MTP file is not discovered262    {"unsloth suffix mtp", unsloth, "test/repo:Q8_K_XL", "", true, false,263     "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"},264     "mmproj-BF16.gguf", "", "", "", ""},265 266    // vendor prefixes and the dot quant convention both match the tag,267    // first match wins between two files at the same quant268    {"vendor prefix", vendors, "test/repo:Q8_0", "", false, true,269     "TheDrummer_Model-24B-v4.1-Q8_0.gguf", {"TheDrummer_Model-24B-v4.1-Q8_0.gguf"},270     "", "", "", "", ""},271 272    // every sidecar type resolves at the tag273    {"quad exact tag", quad, "test/repo:Q8_0", "", true, false,274     "model-Q8_0.gguf", {"model-Q8_0.gguf"},275     "", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "eagle3-model-Q8_0.gguf", "dspark-model-Q8_0.gguf"},276 277    // no tag anchors the dspark sidecar on the only full quant278    {"spark default anchor", spark, "test/repo", "", true, false,279     "model-MXFP4.gguf", {"model-MXFP4.gguf"},280     "", "", "", "", "dspark-model-MXFP4.gguf"},281 282    // a tag with no matching full model still resolves the exact dspark sidecar283    {"spark tag sidecar", spark, "test/repo:BF16", "", true, false,284     "", {},285     "", "", "", "", "dspark-model-BF16.gguf"},286};287 288static void check_plan(const plan_case & c) {289    common_download_opts opts;290    opts.download_mmproj = c.sidecars;291    opts.download_mtp    = c.sidecars;292    opts.download_dflash = c.sidecars;293    opts.download_eagle3 = c.sidecars;294    opts.download_dspark = c.sidecars;295 296    auto plan = common_download_get_hf_plan(model_ref(c.hf_repo, c.hf_file), opts);297 298    REQUIRE_EQ(plan.primary.path, c.primary);299    REQUIRE_EQ(plan.mmproj.path,  c.mmproj);300    REQUIRE_EQ(plan.mtp.path,     c.mtp);301    REQUIRE_EQ(plan.dflash.path,  c.dflash);302    REQUIRE_EQ(plan.eagle3.path,  c.eagle3);303    REQUIRE_EQ(plan.dspark.path,  c.dspark);304 305    // exact shard set, order insensitive; the primary must be the first split306    std::vector<std::string> actual;307    for (const auto & f : plan.model_files) {308        actual.push_back(f.path);309    }310    std::sort(actual.begin(), actual.end());311    auto expected = c.model_files;312    std::sort(expected.begin(), expected.end());313    REQUIRE(actual == expected);314    if (!expected.empty()) {315        REQUIRE(plan.primary.path == expected.front());316    }317}318 319static void test_plan_resolution() {320    printf("test-model-resolution: plan resolution on %zu cases\n", sizeof(plan_cases) / sizeof(plan_cases[0]));321 322    for (const auto & c : plan_cases) {323        printf("  %s\n", c.name);324        // invariant: the resolution is insensitive to the listing order325        for (size_t rot = 0; rot < c.files.size(); ++rot) {326            if (c.order_dependent && rot > 0) {327                continue;328            }329            g_context = std::string(c.name) + ", reordering " + std::to_string(rot);330            auto files = c.files;331            std::rotate(files.begin(), files.begin() + rot, files.end());332            if (rot % 2 == 1) {333                std::reverse(files.begin(), files.end());334            }335            g_repos["test/repo"] = files;336            check_plan(c);337        }338    }339    g_repos.clear();340}341 342//343// end-to-end assembly: real CLI parsing, real handler init resolving over the344// loopback, downloads skipped by flipping offline before apply345//346 347static void assemble(std::vector<std::string> argv, common_params & params) {348    std::vector<char *> cargv;349    g_context.clear();350    for (auto & a : argv) {351        g_context += g_context.empty() ? a : " " + a;352        cargv.push_back(a.data());353    }354    bool ok = common_params_parse((int) cargv.size(), cargv.data(), params, LLAMA_EXAMPLE_SERVER);355    REQUIRE(ok);356 357    auto handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);358 359    // skip the network execution, on_done still wires the params360    params.offline = true;361    common_models_handler_apply(handler, params);362}363 364static void test_task_assembly() {365    printf("test-model-resolution: end-to-end assembly\n");366 367    g_repos["test/main"]   = flat;368    g_repos["test/hole"]   = hole;369    g_repos["test/quad"]   = quad;370    g_repos["test/dflash"] = dflash_only;371    g_repos["test/eagle3"] = eagle3_only;372    g_repos["test/spark"]  = spark;373    g_repos["test/pair"]   = dspark_dflash;374    g_repos["test/small"]  = {"draft-model-Q4_K_M.gguf"};375    g_repos["test/preset"] = {"preset.ini", "model-Q8_0.gguf"};376 377    {378        // plain -hf wires the model and its mmproj, nothing speculative379        common_params params;380        assemble({"server", "-hf", "test/main:Q8_0"}, params);381        REQUIRE_EQ(params.model.path,  cached("test/main", "model-Q8_0.gguf"));382        REQUIRE_EQ(params.mmproj.path, cached("test/main", "mmproj-model-Q8_0.gguf"));383        REQUIRE(params.speculative.draft.mparams.path.empty());384    }385    {386        // --no-mmproj disables the mmproj discovery387        common_params params;388        assemble({"server", "-hf", "test/main:Q8_0", "--no-mmproj"}, params);389        REQUIRE(params.mmproj.path.empty());390    }391    {392        // an explicit --mmproj wins over the discovery393        common_params params;394        assemble({"server", "-hf", "test/main:Q8_0", "--mmproj", "/local/mmproj.gguf"}, params);395        REQUIRE(params.mmproj.path == "/local/mmproj.gguf");396    }397    {398        // -hf with a spec type wires the sidecar of the main repo as fallback draft399        common_params params;400        assemble({"server", "-hf", "test/main:Q8_0", "--spec-type", "draft-mtp"}, params);401        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-Q8_0.gguf"));402    }403    {404        // -hfd with a spec type wires the draft repo sidecar at its tag,405        // not its full model, and suppresses the main repo fallback406        common_params params;407        assemble({"server", "-hf", "test/hole:Q8_0", "-hfd", "test/hole:Q4_0", "--spec-type", "draft-mtp"}, params);408        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/hole", "mtp-model-Q4_0.gguf"));409    }410    {411        // an explicit -md file wins over the sidecar resolution412        common_params params;413        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/main", "-md", "mtp-model-BF16.gguf", "--spec-type", "draft-mtp"}, params);414        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/main", "mtp-model-BF16.gguf"));415    }416    {417        // -hfd without a spec type auto-selects the type, mtp first when all ship418        common_params params;419        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/quad:Q8_0"}, params);420        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_MTP});421        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/quad", "mtp-model-Q8_0.gguf"));422    }423    {424        // auto-selection with only a dflash sidecar425        common_params params;426        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/dflash:Q8_0"}, params);427        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH});428        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/dflash", "dflash-model-Q8_0.gguf"));429    }430    {431        // auto-selection with only an eagle3 sidecar432        common_params params;433        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/eagle3:Q8_0"}, params);434        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3});435        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/eagle3", "eagle3-model-Q8_0.gguf"));436    }437    {438        // auto-selection prefers dspark over dflash when both ship439        common_params params;440        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/pair:Q8_0"}, params);441        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK});442        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/pair", "dspark-model-Q8_0.gguf"));443    }444    {445        // -hf with the dspark spec type wires the sidecar of the main repo,446        // anchored on the only full quant447        common_params params;448        assemble({"server", "-hf", "test/spark", "--spec-type", "draft-dspark"}, params);449        REQUIRE_EQ(params.model.path, cached("test/spark", "model-MXFP4.gguf"));450        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/spark", "dspark-model-MXFP4.gguf"));451    }452    {453        // -hfd on a repo without sidecars keeps resolving a full model as draft454        common_params params;455        assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/small"}, params);456        REQUIRE(params.speculative.types == std::vector<enum common_speculative_type>{COMMON_SPECULATIVE_TYPE_NONE});457        REQUIRE_EQ(params.speculative.draft.mparams.path, cached("test/small", "draft-model-Q4_K_M.gguf"));458    }459    {460        // a preset repo wires the preset and clears the model for router mode461        common_params params;462        assemble({"server", "-hf", "test/preset"}, params);463        REQUIRE_EQ(params.models_preset, cached("test/preset", "preset.ini"));464        REQUIRE(params.model.path.empty());465        REQUIRE(params.model.hf_repo.empty());466    }467 468    g_repos.clear();469}470 471int main(void) {472    // unbuffered, so a crash cannot swallow the reports already printed473    setvbuf(stdout, nullptr, _IONBF, 0);474    setvbuf(stderr, nullptr, _IONBF, 0);475 476    // the negative cases legitimately log errors on every reordering,477    // keep the output down to the reports478    common_log_pause(common_log_main());479 480    // the loopback endpoint also keeps the client init from rejecting481    // https on the builds without TLS support482    httplib::Server server;483    serve_repos(server);484    int port = server.bind_to_any_port("127.0.0.1");485 486    // isolate the cache, its location is read once so it is set487    // before anything else488    cache_dir = std::filesystem::temp_directory_path() /489                ("test-model-resolution-cache-" + std::to_string(port));490    std::filesystem::remove_all(cache_dir);491    common_set_env("LLAMA_CACHE", cache_dir.string());492 493    std::thread server_thread([&server] { server.listen_after_bind(); });494    server.wait_until_ready();495    common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/");496 497    test_plan_resolution();498    test_task_assembly();499 500    server.stop();501    server_thread.join();502 503    std::filesystem::remove_all(cache_dir);504    printf("test-model-resolution: all tests OK\n");505    return 0;506}507