echodict/llama.cpp

1#include "server-context.h"2#include "server-http.h"3#include "server-models.h"4#include "server-cors-proxy.h"5#include "server-tools.h"6 7#include "arg.h"8#include "build-info.h"9#include "common.h"10#include "fit.h"11#include "llama.h"12#include "log.h"13 14#include <atomic>15#include <clocale>16#include <exception>17#include <signal.h>18#include <thread> // for std::thread::hardware_concurrency19 20#if defined(_WIN32)21#include <windows.h>22#endif23 24static std::function<void(int)> shutdown_handler;25static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;26 27static inline void signal_handler(int signal) {28    if (is_terminating.test_and_set()) {29        // in case it hangs, we can force terminate the server by hitting Ctrl+C twice30        // this is for better developer experience, we can remove when the server is stable enough31        fprintf(stderr, "Received second interrupt, terminating immediately.\n");32        exit(1);33    }34 35    shutdown_handler(signal);36}37 38// wrapper function that handles exceptions and logs errors39// this is to make sure handler_t never throws exceptions; instead, it returns an error response40static server_http_context::handler_t ex_wrapper(server_http_context::handler_t func) {41    return [func = std::move(func)](const server_http_req & req) -> server_http_res_ptr {42        std::string message;43        error_type error;44        try {45            return func(req);46        } catch (const std::invalid_argument & e) {47            // treat invalid_argument as invalid request (400)48            error = ERROR_TYPE_INVALID_REQUEST;49            message = e.what();50        } catch (const std::exception & e) {51            // treat other exceptions as server error (500)52            error = ERROR_TYPE_SERVER;53            message = e.what();54        } catch (...) {55            error = ERROR_TYPE_SERVER;56            message = "unknown error";57        }58 59        auto res = std::make_unique<server_http_res>();60        res->status = 500;61        try {62            json error_data = format_error_response(message, error);63            res->status = json_value(error_data, "code", 500);64            res->data = safe_json_to_str({{ "error", error_data }});65            SRV_WRN("got exception: %s\n", res->data.c_str());66        } catch (const std::exception & e) {67            SRV_ERR("got another exception: %s | while handling exception: %s\n", e.what(), message.c_str());68            res->data = "Internal Server Error";69        }70        return res;71    };72}73 74int main(int argc, char ** argv) {75    std::setlocale(LC_NUMERIC, "C");76 77    // own arguments required by this example78    common_params params;79 80    common_init();81 82    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {83        return 1;84    }85 86    // validate batch size for embeddings87    // embeddings require all tokens to be processed in a single ubatch88    // see https://github.com/ggml-org/llama.cpp/issues/1283689    if (params.embedding && params.n_batch > params.n_ubatch) {90        LOG_WRN("%s: embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", __func__, params.n_batch, params.n_ubatch);91        LOG_WRN("%s: setting n_batch = n_ubatch = %d to avoid assertion failure\n", __func__, params.n_ubatch);92        params.n_batch = params.n_ubatch;93    }94 95    if (params.n_parallel < 0) {96        LOG_INF("%s: n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n", __func__);97 98        params.n_parallel = 4;99        params.kv_unified = true;100    }101 102    // for consistency between server router mode and single-model mode, we set the same model name as alias103    if (params.model_alias.empty() && !params.model.name.empty()) {104        params.model_alias.insert(params.model.name);105    }106 107    // struct that contains llama context and inference108    server_context ctx_server;109 110    llama_backend_init();111    llama_numa_init(params.numa);112 113    LOG_INF("build_info: %s\n", llama_build_info());114    LOG_INF("%s\n", common_params_get_system_info(params).c_str());115 116    server_http_context ctx_http;117    if (!ctx_http.init(params)) {118        LOG_ERR("%s: failed to initialize HTTP server\n", __func__);119        return 1;120    }121 122    //123    // Router124    //125 126    // register API routes127    server_routes routes(params, ctx_server);128    server_tools tools;129 130    bool is_router_server = params.model.path.empty();131    std::optional<server_models_routes> models_routes{};132    if (is_router_server) {133        // setup server instances manager134        try {135            models_routes.emplace(params, argc, argv);136        } catch (const std::exception & e) {137            LOG_ERR("%s: failed to initialize router models: %s\n", __func__, e.what());138            return 1;139        }140 141        // proxy handlers142        // note: routes.get_health stays the same143        routes.get_metrics                 = models_routes->proxy_get;144        routes.post_props                  = models_routes->proxy_post;145        routes.post_completions            = models_routes->proxy_post;146        routes.post_completions_oai        = models_routes->proxy_post;147        routes.post_chat_completions       = models_routes->proxy_post;148        routes.post_responses_oai          = models_routes->proxy_post;149        routes.post_transcriptions_oai     = models_routes->proxy_post;150        routes.post_anthropic_messages     = models_routes->proxy_post;151        routes.post_anthropic_count_tokens = models_routes->proxy_post;152        routes.post_infill                 = models_routes->proxy_post;153        routes.post_embeddings             = models_routes->proxy_post;154        routes.post_embeddings_oai         = models_routes->proxy_post;155        routes.post_rerank                 = models_routes->proxy_post;156        routes.post_tokenize               = models_routes->proxy_post;157        routes.post_detokenize             = models_routes->proxy_post;158        routes.post_apply_template         = models_routes->proxy_post;159        routes.get_lora_adapters           = models_routes->proxy_get;160        routes.post_lora_adapters          = models_routes->proxy_post;161        routes.get_slots                   = models_routes->proxy_get;162        routes.post_slots                  = models_routes->proxy_post;163 164        // custom routes for router165        routes.get_props                   = models_routes->get_router_props;166        routes.get_models                  = models_routes->get_router_models;167 168        ctx_http.post("/models/load",          ex_wrapper(models_routes->post_router_models_load));169        ctx_http.post("/models/unload",        ex_wrapper(models_routes->post_router_models_unload));170    }171 172    ctx_http.get ("/health",                   ex_wrapper(routes.get_health)); // public endpoint (no API key check)173    ctx_http.get ("/v1/health",                ex_wrapper(routes.get_health)); // public endpoint (no API key check)174    ctx_http.get ("/metrics",                  ex_wrapper(routes.get_metrics));175    ctx_http.get ("/props",                    ex_wrapper(routes.get_props));176    ctx_http.post("/props",                    ex_wrapper(routes.post_props));177    ctx_http.get ("/models",                   ex_wrapper(routes.get_models)); // public endpoint (no API key check)178    ctx_http.get ("/v1/models",                ex_wrapper(routes.get_models)); // public endpoint (no API key check)179    ctx_http.post("/completion",               ex_wrapper(routes.post_completions)); // legacy180    ctx_http.post("/completions",              ex_wrapper(routes.post_completions));181    ctx_http.post("/v1/completions",           ex_wrapper(routes.post_completions_oai));182    ctx_http.post("/chat/completions",         ex_wrapper(routes.post_chat_completions));183    ctx_http.post("/v1/chat/completions",      ex_wrapper(routes.post_chat_completions));184    ctx_http.post("/v1/responses",             ex_wrapper(routes.post_responses_oai));185    ctx_http.post("/responses",                ex_wrapper(routes.post_responses_oai));186    ctx_http.post("/v1/audio/transcriptions",  ex_wrapper(routes.post_transcriptions_oai));187    ctx_http.post("/audio/transcriptions",     ex_wrapper(routes.post_transcriptions_oai));188    ctx_http.post("/v1/messages",              ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API189    ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting190    ctx_http.post("/infill",                   ex_wrapper(routes.post_infill));191    ctx_http.post("/embedding",                ex_wrapper(routes.post_embeddings)); // legacy192    ctx_http.post("/embeddings",               ex_wrapper(routes.post_embeddings));193    ctx_http.post("/v1/embeddings",            ex_wrapper(routes.post_embeddings_oai));194    ctx_http.post("/rerank",                   ex_wrapper(routes.post_rerank));195    ctx_http.post("/reranking",                ex_wrapper(routes.post_rerank));196    ctx_http.post("/v1/rerank",                ex_wrapper(routes.post_rerank));197    ctx_http.post("/v1/reranking",             ex_wrapper(routes.post_rerank));198    ctx_http.post("/tokenize",                 ex_wrapper(routes.post_tokenize));199    ctx_http.post("/detokenize",               ex_wrapper(routes.post_detokenize));200    ctx_http.post("/apply-template",           ex_wrapper(routes.post_apply_template));201    // LoRA adapters hotswap202    ctx_http.get ("/lora-adapters",            ex_wrapper(routes.get_lora_adapters));203    ctx_http.post("/lora-adapters",            ex_wrapper(routes.post_lora_adapters));204    // Save & load slots205    ctx_http.get ("/slots",                    ex_wrapper(routes.get_slots));206    ctx_http.post("/slots/:id_slot",           ex_wrapper(routes.post_slots));207    // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)208    if (params.webui_mcp_proxy) {209        SRV_WRN("%s", "-----------------\n");210        SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n");211        SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n");212        SRV_WRN("%s", "-----------------\n");213        ctx_http.get ("/cors-proxy",      ex_wrapper(proxy_handler_get));214        ctx_http.post("/cors-proxy",      ex_wrapper(proxy_handler_post));215    }216    // EXPERIMENTAL built-in tools217    if (!params.server_tools.empty()) {218        tools.setup(params.server_tools);219        SRV_WRN("%s", "-----------------\n");220        SRV_WRN("%s", "Built-in tools are enabled, do not expose server to untrusted environments\n");221        SRV_WRN("%s", "This feature is EXPERIMENTAL and may be changed in the future\n");222        SRV_WRN("%s", "-----------------\n");223        ctx_http.get ("/tools",           ex_wrapper(tools.handle_get));224        ctx_http.post("/tools",           ex_wrapper(tools.handle_post));225    }226 227    //228    // Start the server229    //230 231    std::function<void()> clean_up;232 233    if (is_router_server) {234        LOG_INF("%s: starting router server, no model will be loaded in this process\n", __func__);235 236        clean_up = [&models_routes]() {237            SRV_INF("%s: cleaning up before exit...\n", __func__);238            if (models_routes.has_value()) {239                models_routes->models.unload_all();240            }241            llama_backend_free();242        };243 244        if (!ctx_http.start()) {245            clean_up();246            LOG_ERR("%s: exiting due to HTTP server error\n", __func__);247            return 1;248        }249        ctx_http.is_ready.store(true);250 251        shutdown_handler = [&](int) {252            ctx_http.stop();253        };254 255    } else {256        // setup clean up function, to be called before exit257        clean_up = [&ctx_http, &ctx_server]() {258            SRV_INF("%s: cleaning up before exit...\n", __func__);259            ctx_http.stop();260            ctx_server.terminate();261            llama_backend_free();262        };263 264        // start the HTTP server before loading the model to be able to serve /health requests265        if (!ctx_http.start()) {266            clean_up();267            LOG_ERR("%s: exiting due to HTTP server error\n", __func__);268            return 1;269        }270 271        // load the model272        LOG_INF("%s: loading model\n", __func__);273 274        if (server_models::is_child_server()) {275            ctx_server.on_sleeping_changed([&](bool sleeping) {276                server_models::notify_router_sleeping_state(sleeping);277            });278        }279 280        if (!ctx_server.load_model(params)) {281            clean_up();282            if (ctx_http.thread.joinable()) {283                ctx_http.thread.join();284            }285            LOG_ERR("%s: exiting due to model loading error\n", __func__);286            return 1;287        }288 289        routes.update_meta(ctx_server);290        ctx_http.is_ready.store(true);291 292        LOG_INF("%s: model loaded\n", __func__);293 294        shutdown_handler = [&](int) {295            // this will unblock start_loop()296            ctx_server.terminate();297        };298    }299 300    // TODO: refactor in common/console301#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))302    struct sigaction sigint_action;303    sigint_action.sa_handler = signal_handler;304    sigemptyset (&sigint_action.sa_mask);305    sigint_action.sa_flags = 0;306    sigaction(SIGINT, &sigint_action, NULL);307    sigaction(SIGTERM, &sigint_action, NULL);308#elif defined (_WIN32)309    auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {310        return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;311    };312    SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);313#endif314 315    if (is_router_server) {316        LOG_INF("%s: router server is listening on %s\n", __func__, ctx_http.listening_address.c_str());317        LOG_INF("%s: NOTE: router mode is experimental\n", __func__);318        LOG_INF("%s:       it is not recommended to use this mode in untrusted environments\n", __func__);319        if (ctx_http.thread.joinable()) {320            ctx_http.thread.join(); // keep the main thread alive321        }322 323        // when the HTTP server stops, clean up and exit324        clean_up();325    } else {326        LOG_INF("%s: server is listening on %s\n", __func__, ctx_http.listening_address.c_str());327        LOG_INF("%s: starting the main loop...\n", __func__);328 329        // optionally, notify router server that this instance is ready330        std::thread monitor_thread;331        if (server_models::is_child_server()) {332            monitor_thread = server_models::setup_child_server(shutdown_handler);333        }334 335        // this call blocks the main thread until queue_tasks.terminate() is called336        ctx_server.start_loop();337 338        clean_up();339        if (ctx_http.thread.joinable()) {340            ctx_http.thread.join();341        }342        if (monitor_thread.joinable()) {343            monitor_thread.join();344        }345 346        auto * ll_ctx = ctx_server.get_llama_context();347        if (ll_ctx != nullptr) {348            common_memory_breakdown_print(ll_ctx);349        }350    }351 352    return 0;353}354