Felipe97/llama-cpp-compiled

1#include "server-context.h"2#include "server-http.h"3#include "server-models.h"4#include "server-cors-proxy.h"5#include "server-stream.h"6#include "server-tools.h"7 8#include "arg.h"9#include "build-info.h"10#include "common.h"11#include "fit.h"12#include "llama.h"13#include "log.h"14 15#include <atomic>16#include <clocale>17#include <exception>18#include <signal.h>19#include <thread> // for std::thread::hardware_concurrency20 21#if defined(_WIN32)22#include <windows.h>23#endif24 25static std::function<void(int)> shutdown_handler;26static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;27 28static inline void signal_handler(int signal) {29    if (is_terminating.test_and_set()) {30        // in case it hangs, we can force terminate the server by hitting Ctrl+C twice31        // this is for better developer experience, we can remove when the server is stable enough32        fprintf(stderr, "Received second interrupt, terminating immediately.\n");33        exit(1);34    }35 36    shutdown_handler(signal);37}38 39// satisfies -Wmissing-declarations (used by llama command)40int llama_server(int argc, char ** argv);41 42// to be used via CLI (argc / argv are used by router mode only)43int llama_server(common_params & params, int argc, char ** argv);44void llama_server_terminate();45void llama_server_terminate() {46    if (shutdown_handler) {47        shutdown_handler(0);48    }49}50 51 52// wrapper function that handles exceptions and logs errors53// this is to make sure handler_t never throws exceptions; instead, it returns an error response54static server_http_context::handler_t ex_wrapper(server_http_context::handler_t func) {55    return [func = std::move(func)](const server_http_req & req) -> server_http_res_ptr {56        std::string message;57        error_type error;58        try {59            return func(req);60        } catch (const std::invalid_argument & e) {61            // treat invalid_argument as invalid request (400)62            error = ERROR_TYPE_INVALID_REQUEST;63            message = e.what();64        } catch (const std::exception & e) {65            // treat other exceptions as server error (500)66            error = ERROR_TYPE_SERVER;67            message = e.what();68        } catch (...) {69            error = ERROR_TYPE_SERVER;70            message = "unknown error";71        }72 73        auto res = std::make_unique<server_http_res>();74        res->status = 500;75        try {76            json error_data = format_error_response(message, error);77            res->status = json_value(error_data, "code", 500);78            res->data = safe_json_to_str({{ "error", error_data }});79            SRV_WRN("got exception: %s\n", res->data.c_str());80        } catch (const std::exception & e) {81            SRV_ERR("got another exception: %s | while handling exception: %s\n", e.what(), message.c_str());82            res->data = "Internal Server Error";83        }84        return res;85    };86}87 88int llama_server(int argc, char ** argv) {89    std::setlocale(LC_NUMERIC, "C");90 91#ifndef _WIN3292    // Ignore SIGPIPE so the server does not crash if a child (MCP server, tools runtime) exits while we are writing to its stdin93    signal(SIGPIPE, SIG_IGN);94#endif95 96    // own arguments required by this example97    common_params params;98 99    common_init();100 101    // start the stream session manager GC right after common init, before any HTTP route can102    // touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free103    server_stream_session_manager_start();104 105    SRV_INF("%s", "initializing ...\n");106 107    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {108        return 1;109    }110 111    llama_backend_init();112    llama_numa_init(params.numa);113 114    return llama_server(params, argc, argv);115}116 117int llama_server(common_params & params, int argc, char ** argv) {118    bool is_run_by_cli = (argv == nullptr);119 120    common_models_handler models_handler;121 122    // note: router mode also accepts -hf remote-preset, so we need to check that first123    if (!is_run_by_cli && !params.model.hf_repo.empty()) {124        try {125            models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);126            if (common_models_handler_is_preset_repo(models_handler)) {127                // apply the preset and start the server in router mode128                common_models_handler_apply(models_handler, params);129            }130        } catch (const std::exception & e) {131            SRV_ERR("failed to fetch model metadata: %s\n", e.what());132            return 1;133        }134    }135 136    // router server never loads a model and must not touch the GPU137    const bool is_router_server = params.model.path.empty()138                               && params.model.hf_repo.empty()139                               && params.model.docker_repo.empty();140 141    // skip device enumeration so the CUDA primary context stays uncreated142    common_params_print_info(params, !is_router_server);143 144    if (!is_router_server) {145        // validate batch size for embeddings146        // embeddings require all tokens to be processed in a single ubatch147        // see https://github.com/ggml-org/llama.cpp/issues/12836148        if (params.embedding && params.n_batch > params.n_ubatch) {149            SRV_WRN("embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", params.n_batch, params.n_ubatch);150            SRV_WRN("setting n_batch = n_ubatch = %d to avoid assertion failure\n", params.n_ubatch);151            params.n_batch = params.n_ubatch;152        }153 154        if (params.n_parallel < 0) {155            SRV_TRC("%s", "n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n");156 157            params.n_parallel = 4;158            params.kv_unified = true;159        }160    }161 162    // size the KV pool from --kv-unified-per-slot, unless the user pinned it with -c163    // or with -c 0 for max context164    const bool ctx_pool_auto_sized = params.kv_unified_per_slot > 0 &&165                                     params.n_ctx == 0 &&166                                     (uint32_t) params.fit_params_min_ctx != UINT32_MAX;167 168    if (ctx_pool_auto_sized) {169        params.n_ctx = params.n_parallel * params.kv_unified_per_slot;170        SRV_INF("--kv-unified-per-slot: sizing KV pool to n_parallel * kv_unified_per_slot = %d * %d = %d\n", params.n_parallel,171                params.kv_unified_per_slot, params.n_ctx);172    }173 174    // for consistency between server router mode and single-model mode, we set the same model name as alias175    auto model_name = params.model.get_name();176    if (params.model_alias.empty() && !model_name.empty()) {177        params.model_alias.insert(model_name);178    }179 180    // note: this is guaranteed to out-live ctx_http and tools181    server_mcp mcp_mgr;182 183    // struct that contains llama context and inference184    server_context ctx_server;185 186    server_http_context ctx_http;187    if (!ctx_http.init(params)) {188        SRV_ERR("%s", "failed to initialize HTTP server\n");189        return 1;190    }191 192    //193    // Router194    //195 196    // register API routes197    server_child child; // only used in non-router mode198    server_routes routes(params, ctx_server);199    server_tools tools;200 201    std::optional<server_models_routes> models_routes{};202    if (is_router_server) {203        // setup server instances manager204        try {205            models_routes.emplace(params, argc, argv);206        } catch (const std::exception & e) {207            SRV_ERR("failed to initialize router models: %s\n", e.what());208            return 1;209        }210 211        // proxy handlers212        // note: routes.get_health stays the same213        routes.get_metrics                 = models_routes->proxy_get;214        routes.post_props                  = models_routes->proxy_post;215        routes.post_completions            = models_routes->proxy_post;216        routes.post_completions_oai        = models_routes->proxy_post;217        routes.post_chat_completions       = models_routes->proxy_post;218        routes.post_control                = models_routes->proxy_post;219        routes.post_responses_oai          = models_routes->proxy_post;220        routes.post_transcriptions_oai     = models_routes->proxy_post;221        routes.post_anthropic_messages     = models_routes->proxy_post;222        routes.post_anthropic_count_tokens = models_routes->proxy_post;223        routes.post_infill                 = models_routes->proxy_post;224        routes.post_embeddings             = models_routes->proxy_post;225        routes.post_embeddings_oai         = models_routes->proxy_post;226        routes.post_rerank                 = models_routes->proxy_post;227        routes.post_tokenize               = models_routes->proxy_post;228        routes.post_detokenize             = models_routes->proxy_post;229        routes.post_apply_template         = models_routes->proxy_post;230        routes.post_chat_completions_tok   = models_routes->proxy_post;231        routes.post_responses_tok_oai      = models_routes->proxy_post;232        routes.get_lora_adapters           = models_routes->proxy_get;233        routes.post_lora_adapters          = models_routes->proxy_post;234        routes.get_slots                   = models_routes->proxy_get;235        routes.post_slots                  = models_routes->proxy_post;236 237        // custom routes for router238        routes.get_props                   = models_routes->get_router_props;239        routes.get_models                  = models_routes->get_router_models;240 241        ctx_http.post("/models",               ex_wrapper(models_routes->post_router_models));242        ctx_http.post("/models/load",          ex_wrapper(models_routes->post_router_models_load));243        ctx_http.post("/models/unload",        ex_wrapper(models_routes->post_router_models_unload));244        ctx_http.get ("/models/sse",           ex_wrapper(models_routes->get_router_models_sse));245        ctx_http.del ("/models",               ex_wrapper(models_routes->del_router_models));246    }247 248    ctx_http.get ("/health",                   ex_wrapper(routes.get_health)); // public endpoint (no API key check)249    ctx_http.get ("/v1/health",                ex_wrapper(routes.get_health)); // public endpoint (no API key check)250    ctx_http.get ("/metrics",                  ex_wrapper(routes.get_metrics));251    ctx_http.get ("/props",                    ex_wrapper(routes.get_props));252    ctx_http.post("/props",                    ex_wrapper(routes.post_props));253    ctx_http.get ("/models",                   ex_wrapper(routes.get_models));254    ctx_http.get ("/v1/models",                ex_wrapper(routes.get_models));255    ctx_http.post("/completion",               ex_wrapper(routes.post_completions)); // legacy256    ctx_http.post("/completions",              ex_wrapper(routes.post_completions));257    ctx_http.post("/v1/completions",           ex_wrapper(routes.post_completions_oai));258    ctx_http.post("/chat/completions",         ex_wrapper(routes.post_chat_completions));259    ctx_http.post("/v1/chat/completions",      ex_wrapper(routes.post_chat_completions));260    ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control));261    ctx_http.post("/v1/responses",             ex_wrapper(routes.post_responses_oai));262    ctx_http.post("/responses",                ex_wrapper(routes.post_responses_oai));263    ctx_http.post("/v1/audio/transcriptions",  ex_wrapper(routes.post_transcriptions_oai));264    ctx_http.post("/audio/transcriptions",     ex_wrapper(routes.post_transcriptions_oai));265    ctx_http.post("/v1/messages",              ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API266    ctx_http.post("/infill",                   ex_wrapper(routes.post_infill));267    ctx_http.post("/embedding",                ex_wrapper(routes.post_embeddings)); // legacy268    ctx_http.post("/embeddings",               ex_wrapper(routes.post_embeddings));269    ctx_http.post("/v1/embeddings",            ex_wrapper(routes.post_embeddings_oai));270    ctx_http.post("/rerank",                   ex_wrapper(routes.post_rerank));271    ctx_http.post("/reranking",                ex_wrapper(routes.post_rerank));272    ctx_http.post("/v1/rerank",                ex_wrapper(routes.post_rerank));273    ctx_http.post("/v1/reranking",             ex_wrapper(routes.post_rerank));274    ctx_http.post("/tokenize",                 ex_wrapper(routes.post_tokenize));275    ctx_http.post("/detokenize",               ex_wrapper(routes.post_detokenize));276    ctx_http.post("/apply-template",           ex_wrapper(routes.post_apply_template));277    // token counting278    ctx_http.post("/chat/completions/input_tokens",    ex_wrapper(routes.post_chat_completions_tok));279    ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));280    ctx_http.post("/responses/input_tokens",           ex_wrapper(routes.post_responses_tok_oai));281    ctx_http.post("/v1/responses/input_tokens",        ex_wrapper(routes.post_responses_tok_oai));282    ctx_http.post("/v1/messages/count_tokens",         ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting283    // LoRA adapters hotswap284    ctx_http.get ("/lora-adapters",            ex_wrapper(routes.get_lora_adapters));285    ctx_http.post("/lora-adapters",            ex_wrapper(routes.post_lora_adapters));286    // Save & load slots287    ctx_http.get ("/slots",                    ex_wrapper(routes.get_slots));288    ctx_http.post("/slots/:id_slot",           ex_wrapper(routes.post_slots));289 290    // resumable streaming: a child binds the local session factories, the router binds291    // proxies that resolve the owning child, see server-stream.h292    server_http_context::handler_t stream_get_h;293    server_http_context::handler_t streams_lookup_h;294    server_http_context::handler_t stream_delete_h;295    if (is_router_server) {296        stream_get_h     = models_routes->router_stream_get;297        streams_lookup_h = models_routes->router_streams_lookup;298        stream_delete_h  = models_routes->router_stream_delete;299    } else {300        stream_get_h     = server_stream_make_get_handler();301        streams_lookup_h = server_stream_make_lookup_handler();302        stream_delete_h  = server_stream_make_delete_handler();303    }304    ctx_http.get ("/v1/stream",                ex_wrapper(stream_get_h));305    ctx_http.post("/v1/streams/lookup",        ex_wrapper(streams_lookup_h));306    ctx_http.del ("/v1/stream",                ex_wrapper(stream_delete_h));307 308    // Google Cloud Platform (Vertex AI) compat309    ctx_http.register_gcp_compat();310 311    // return 403 for disabled features312    server_http_context::handler_t res_403 = [](const server_http_req &) {313        auto res = std::make_unique<server_http_res>();314        res->status = 403;315        res->data = safe_json_to_str({316            {"error", {317                {"message", "this feature is disabled"},318                {"type", "feature_disabled"},319            }}320        });321        return res;322    };323 324    if (params.cors_origins == "*" && params.api_keys.empty()) {325        SRV_WRN("%s", "security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655)\n");326    }327 328    // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)329    std::vector<std::string> warn_names;330    if (is_router_server) {331        warn_names.push_back("router mode");332    }333 334    if (params.ui_mcp_proxy) {335        ctx_http.get ("/cors-proxy",      ex_wrapper(proxy_handler_get));336        ctx_http.post("/cors-proxy",      ex_wrapper(proxy_handler_post));337        warn_names.push_back("MCP proxy (experimental)");338    } else {339        ctx_http.get ("/cors-proxy",      ex_wrapper(res_403));340        ctx_http.post("/cors-proxy",      ex_wrapper(res_403));341    }342 343    try {344        mcp_mgr.start(params);345    } catch (const std::exception & e) {346        SRV_ERR("MCP starting failed: %s\n", e.what());347        return 1;348    }349 350    if (!params.server_tools.empty() || !mcp_mgr.empty()) {351        try {352            tools.setup(params.server_tools, mcp_mgr, params.server_tools_runtime);353        } catch (const std::exception & e) {354            SRV_ERR("tools setup failed: %s\n", e.what());355            return 1;356        }357        ctx_http.get ("/tools",           ex_wrapper(tools.handle_get));358        ctx_http.post("/tools",           ex_wrapper(tools.handle_post));359        if (!params.server_tools.empty()) {360            warn_names.push_back("server tools (experimental)");361        }362        if (!params.server_tools_runtime.empty()) {363            warn_names.push_back("tools runtime (experimental)");364        }365        if (!mcp_mgr.empty()) {366            warn_names.push_back("MCP servers (experimental)");367        }368    } else {369        ctx_http.get ("/tools",           ex_wrapper(res_403));370        ctx_http.post("/tools",           ex_wrapper(res_403));371    }372 373    if (!warn_names.empty()) {374        std::string features;375        for (const auto & name : warn_names) {376            if (!features.empty()) features += ", ";377            features += name;378        }379        SRV_WRN("security: %s enabled - do not expose to untrusted environments\n", features.c_str());380    }381 382    //383    // Handle downloading model384    //385 386    if (child.is_child() && child.get_mode() == SERVER_CHILD_MODE_DOWNLOAD) {387        return child.run_download(params);388    } else if (!is_router_server && !is_run_by_cli) {389        // single-model mode (NOT spawned by router)390        // if this is invoked by CLI, model downloading should be already handled391        try {392            common_models_handler_apply(models_handler, params);393        } catch (const std::exception & e) {394            SRV_ERR("failed to download model: %s\n", e.what());395            return 1;396        }397    }398 399    //400    // Start the server401    //402 403    std::function<void()> clean_up;404 405    if (is_router_server) {406        SRV_INF("%s", "starting server in router mode. models will be automatically loaded on-demand\n");407 408        clean_up = [&models_routes, &mcp_mgr]() {409            SRV_INF("%s: cleaning up before exit...\n", __func__);410            // stop the session GC first, it finalizes live sessions and wakes pending readers411            server_stream_session_manager_stop();412            if (models_routes.has_value()) {413                models_routes->stopping.store(true); // maybe redundant, but just to be safe414                models_routes->models.unload_all();415            }416            mcp_mgr.shutdown();417            llama_backend_free();418        };419 420        if (!ctx_http.start()) {421            clean_up();422            SRV_ERR("%s", "exiting due to HTTP server error\n");423            return 1;424        }425        ctx_http.is_ready.store(true);426 427        shutdown_handler = [&](int) {428            if (models_routes.has_value()) {429                // important to disconnect any SSE clients430                models_routes->stopping.store(true);431            }432            mcp_mgr.shutdown();433            ctx_http.stop();434        };435 436        try {437            models_routes->models.load_startup_models();438        } catch (const std::exception & e) {439            SRV_ERR("failed to load models on startup: %s\n", e.what());440            ctx_http.stop();441            if (ctx_http.thread.joinable()) {442                ctx_http.thread.join();443            }444            clean_up();445            return 1;446        }447 448    } else {449        // setup clean up function, to be called before exit450        clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {451            SRV_INF("%s: cleaning up before exit...\n", __func__);452            // stop the session GC first, it finalizes live sessions and wakes pending readers453            server_stream_session_manager_stop();454            ctx_http.stop();455            ctx_server.terminate();456            mcp_mgr.shutdown();457            llama_backend_free();458        };459 460        // start the HTTP server before loading the model to be able to serve /health requests461        if (!ctx_http.start()) {462            clean_up();463            SRV_ERR("%s", "exiting due to HTTP server error\n");464            return 1;465        }466 467        // setup communication child --> router if necessary468        if (child.is_child()) {469            ctx_server.set_state_callback([&](server_state state, json payload) {470                child.notify_to_router(server_state_to_str(state), payload);471            });472        }473 474        if (!ctx_server.load_model(params)) {475            clean_up();476            if (ctx_http.thread.joinable()) {477                ctx_http.thread.join();478            }479            SRV_ERR("%s", "exiting due to model loading error\n");480            return 1;481        }482 483        routes.update_meta(ctx_server);484        ctx_http.is_ready.store(true);485 486        SRV_INF("%s", "model loaded\n");487 488        shutdown_handler = [&](int) {489            mcp_mgr.shutdown();490            // this will unblock start_loop()491            ctx_server.terminate();492        };493    }494 495    // register signal handler if not running by CLI496    if (!is_run_by_cli) {497#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))498        struct sigaction sigint_action;499        sigint_action.sa_handler = signal_handler;500        sigemptyset (&sigint_action.sa_mask);501        sigint_action.sa_flags = 0;502        sigaction(SIGINT, &sigint_action, NULL);503        sigaction(SIGTERM, &sigint_action, NULL);504#elif defined (_WIN32)505        auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {506            return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;507        };508        SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);509#endif510    }511 512    SRV_INF("listening on %s\n", ctx_http.listening_address.c_str());513 514    // TODO: remove this in the future515    // check the string to also handle the .sock case516    if (string_ends_with(ctx_http.listening_address, ":8080")) {517        SRV_WRN("%s", "notice: server default port will be changed to :9931 in a future release (ref: https://github.com/ggml-org/llama.cpp/pull/26508)\n");518    }519 520    if (is_router_server) {521        if (!params.models_preset_hf.empty()) {522            SRV_WRN(      "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());523            SRV_WRN("%s", "      please only use presets that you can trust! Unknown presets may be unsafe\n");524        }525 526        if (ctx_http.thread.joinable()) {527            ctx_http.thread.join(); // keep the main thread alive528        }529 530        // when the HTTP server stops, clean up and exit531        clean_up();532    } else {533        // optionally, notify router server that this instance is ready534        std::thread monitor_thread;535        if (child.is_child()) {536            monitor_thread = child.setup(shutdown_handler);537            child.notify_to_router(server_state_to_str(SERVER_STATE_READY), routes.get_model_info());538        }539 540        // this call blocks the main thread until queue_tasks.terminate() is called541        ctx_server.start_loop();542 543        clean_up();544        if (ctx_http.thread.joinable()) {545            ctx_http.thread.join();546        }547        if (monitor_thread.joinable()) {548            monitor_thread.join();549        }550 551        auto * ll_ctx = ctx_server.get_llama_context();552        if (ll_ctx != nullptr) {553            common_memory_breakdown_print(ll_ctx);554        }555    }556 557    return 0;558}559