echodict/llama.cpp

1#include "common.h"2#include "server-http.h"3#include "server-common.h"4 5#include <cpp-httplib/httplib.h>6 7#include <functional>8#include <string>9#include <thread>10 11#ifdef LLAMA_BUILD_WEBUI12// auto generated files (see README.md for details)13#include "index.html.hpp"14#include "bundle.js.hpp"15#include "bundle.css.hpp"16#include "loading.html.hpp"17#endif18 19//20// HTTP implementation using cpp-httplib21//22 23class server_http_context::Impl {24public:25    std::unique_ptr<httplib::Server> srv;26};27 28server_http_context::server_http_context()29    : pimpl(std::make_unique<server_http_context::Impl>())30{}31 32server_http_context::~server_http_context() = default;33 34static void log_server_request(const httplib::Request & req, const httplib::Response & res) {35    // skip logging requests that are regularly sent, to avoid log spam36    if (req.path == "/health"37        || req.path == "/v1/health"38        || req.path == "/models"39        || req.path == "/v1/models"40        || req.path == "/props"41        || req.path == "/metrics"42    ) {43        return;44    }45 46    // reminder: this function is not covered by httplib's exception handler; if someone does more complicated stuff, think about wrapping it in try-catch47 48    SRV_INF("done request: %s %s %s %d\n", req.method.c_str(), req.path.c_str(), req.remote_addr.c_str(), res.status);49 50    SRV_DBG("request:  %s\n", req.body.c_str());51    SRV_DBG("response: %s\n", res.body.c_str());52}53 54bool server_http_context::init(const common_params & params) {55    path_prefix = params.api_prefix;56    port = params.port;57    hostname = params.hostname;58 59    auto & srv = pimpl->srv;60 61#ifdef CPPHTTPLIB_OPENSSL_SUPPORT62    if (params.ssl_file_key != "" && params.ssl_file_cert != "") {63        LOG_INF("Running with SSL: key = %s, cert = %s\n", params.ssl_file_key.c_str(), params.ssl_file_cert.c_str());64        srv.reset(65            new httplib::SSLServer(params.ssl_file_cert.c_str(), params.ssl_file_key.c_str())66        );67    } else {68        LOG_INF("Running without SSL\n");69        srv.reset(new httplib::Server());70    }71#else72    if (params.ssl_file_key != "" && params.ssl_file_cert != "") {73        LOG_ERR("Server is built without SSL support\n");74        return false;75    }76    srv.reset(new httplib::Server());77#endif78 79    srv->set_default_headers({{"Server", "llama.cpp"}});80    srv->set_logger(log_server_request);81    srv->set_exception_handler([](const httplib::Request &, httplib::Response & res, const std::exception_ptr & ep) {82        // this is fail-safe; exceptions should already handled by `ex_wrapper`83 84        std::string message;85        try {86            std::rethrow_exception(ep);87        } catch (const std::exception & e) {88            message = e.what();89        } catch (...) {90            message = "Unknown Exception";91        }92 93        res.status = 500;94        res.set_content(message, "text/plain");95        LOG_ERR("got exception: %s\n", message.c_str());96    });97 98    srv->set_error_handler([](const httplib::Request &, httplib::Response & res) {99        if (res.status == 404) {100            res.set_content(101                safe_json_to_str(json {102                    {"error", {103                        {"message", "File Not Found"},104                        {"type", "not_found_error"},105                        {"code", 404}106                    }}107                }),108                "application/json; charset=utf-8"109            );110        }111        // for other error codes, we skip processing here because it's already done by res->error()112    });113 114    // set timeouts and change hostname and port115    srv->set_read_timeout (params.timeout_read);116    srv->set_write_timeout(params.timeout_write);117    srv->set_socket_options([reuse_port = params.reuse_port](socket_t sock) {118        httplib::set_socket_opt(sock, SOL_SOCKET, SO_REUSEADDR, 1);119        if (reuse_port) {120#ifdef SO_REUSEPORT121            httplib::set_socket_opt(sock, SOL_SOCKET, SO_REUSEPORT, 1);122#else123            LOG_WRN("%s: SO_REUSEPORT is not supported\n", __func__);124#endif125        }126    });127 128    if (params.api_keys.size() == 1) {129        auto key = params.api_keys[0];130        std::string substr = key.substr(std::max((int)(key.length() - 4), 0));131        LOG_INF("%s: api_keys: ****%s\n", __func__, substr.c_str());132    } else if (params.api_keys.size() > 1) {133        LOG_INF("%s: api_keys: %zu keys loaded\n", __func__, params.api_keys.size());134    }135 136    //137    // Middlewares138    //139 140    auto middleware_validate_api_key = [api_keys = params.api_keys](const httplib::Request & req, httplib::Response & res) {141        static const std::unordered_set<std::string> public_endpoints = {142            "/health",143            "/v1/health",144            "/models",145            "/v1/models",146            "/",147            "/index.html",148            "/bundle.js",149            "/bundle.css",150        };151 152        // If API key is not set, skip validation153        if (api_keys.empty()) {154            return true;155        }156 157        // If path is public or static file, skip validation158        if (public_endpoints.find(req.path) != public_endpoints.end()) {159            return true;160        }161 162        // Check for API key in the Authorization header163        std::string req_api_key = req.get_header_value("Authorization");164        if (req_api_key.empty()) {165            // retry with anthropic header166            req_api_key = req.get_header_value("X-Api-Key");167        }168 169        // remove the "Bearer " prefix if needed170        std::string prefix = "Bearer ";171        if (req_api_key.substr(0, prefix.size()) == prefix) {172            req_api_key = req_api_key.substr(prefix.size());173        }174 175        // validate the API key176        if (std::find(api_keys.begin(), api_keys.end(), req_api_key) != api_keys.end()) {177            return true; // API key is valid178        }179 180        // API key is invalid or not provided181        res.status = 401;182        res.set_content(183            safe_json_to_str(json {184                {"error", {185                    {"message", "Invalid API Key"},186                    {"type", "authentication_error"},187                    {"code", 401}188                }}189            }),190            "application/json; charset=utf-8"191        );192 193        LOG_WRN("Unauthorized: Invalid API Key\n");194 195        return false;196    };197 198    auto middleware_server_state = [this](const httplib::Request & req, httplib::Response & res) {199        bool ready = is_ready.load();200        if (!ready) {201#ifdef LLAMA_BUILD_WEBUI202            auto tmp = string_split<std::string>(req.path, '.');203            if (req.path == "/" || tmp.back() == "html") {204                res.status = 503;205                res.set_content(reinterpret_cast<const char*>(loading_html), loading_html_len, "text/html; charset=utf-8");206            } else207#endif208            {209                // no endpoints is allowed to be accessed when the server is not ready210                // this is to prevent any data races or inconsistent states211                res.status = 503;212                res.set_content(213                    safe_json_to_str(json {214                        {"error", {215                            {"message", "Loading model"},216                            {"type", "unavailable_error"},217                            {"code", 503}218                        }}219                    }),220                    "application/json; charset=utf-8"221                );222            }223            return false;224        }225        return true;226    };227 228    // register server middlewares229    srv->set_pre_routing_handler([middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {230        res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));231        // If this is OPTIONS request, skip validation because browsers don't include Authorization header232        if (req.method == "OPTIONS") {233            res.set_header("Access-Control-Allow-Credentials", "true");234            res.set_header("Access-Control-Allow-Methods",     "GET, POST");235            res.set_header("Access-Control-Allow-Headers",     "*");236            res.set_content("", "text/html"); // blank response, no data237            return httplib::Server::HandlerResponse::Handled; // skip further processing238        }239        if (!middleware_server_state(req, res)) {240            return httplib::Server::HandlerResponse::Handled;241        }242        if (!middleware_validate_api_key(req, res)) {243            return httplib::Server::HandlerResponse::Handled;244        }245        return httplib::Server::HandlerResponse::Unhandled;246    });247 248    int n_threads_http = params.n_threads_http;249    if (n_threads_http < 1) {250        // +4 threads for monitoring, health and some threads reserved for MCP and other tasks in the future251        n_threads_http = std::max(params.n_parallel + 4, (int32_t) std::thread::hardware_concurrency() - 1);252    }253    LOG_INF("%s: using %d threads for HTTP server\n", __func__, n_threads_http);254    srv->new_task_queue = [n_threads_http] {255        // spawn n_threads_http fixed thread (always alive), while allow up to 1024 max possible additional threads256        // when n_threads_http is used, server will create new "dynamic" threads that will be destroyed after processing each request257        // ref: https://github.com/yhirose/cpp-httplib/pull/2368258        size_t max_threads = (size_t)n_threads_http + 1024;259        return new httplib::ThreadPool(n_threads_http, max_threads);260    };261 262    //263    // Web UI setup264    //265 266    if (!params.webui) {267        LOG_INF("Web UI is disabled\n");268    } else {269        // register static assets routes270        if (!params.public_path.empty()) {271            // Set the base directory for serving static files272            bool is_found = srv->set_mount_point(params.api_prefix + "/", params.public_path);273            if (!is_found) {274                LOG_ERR("%s: static assets path not found: %s\n", __func__, params.public_path.c_str());275                return 1;276            }277        } else {278#ifdef LLAMA_BUILD_WEBUI279            // using embedded static index.html280            srv->Get(params.api_prefix + "/", [](const httplib::Request & /*req*/, httplib::Response & res) {281                // COEP and COOP headers, required by pyodide (python interpreter)282                res.set_header("Cross-Origin-Embedder-Policy", "require-corp");283                res.set_header("Cross-Origin-Opener-Policy", "same-origin");284                res.set_content(reinterpret_cast<const char*>(index_html), index_html_len, "text/html; charset=utf-8");285                return false;286            });287            srv->Get(params.api_prefix + "/bundle.js", [](const httplib::Request & /*req*/, httplib::Response & res) {288                res.set_content(reinterpret_cast<const char*>(bundle_js), bundle_js_len, "application/javascript; charset=utf-8");289                return false;290            });291            srv->Get(params.api_prefix + "/bundle.css", [](const httplib::Request & /*req*/, httplib::Response & res) {292                res.set_content(reinterpret_cast<const char*>(bundle_css), bundle_css_len, "text/css; charset=utf-8");293                return false;294            });295#endif296        }297    }298    return true;299}300 301bool server_http_context::start() {302    // Bind and listen303 304    auto & srv = pimpl->srv;305    bool was_bound = false;306    bool is_sock = false;307    if (string_ends_with(std::string(hostname), ".sock")) {308        is_sock = true;309        LOG_INF("%s: setting address family to AF_UNIX\n", __func__);310        srv->set_address_family(AF_UNIX);311        // bind_to_port requires a second arg, any value other than 0 should312        // simply get ignored313        was_bound = srv->bind_to_port(hostname, 8080);314    } else {315        LOG_INF("%s: binding port with default address family\n", __func__);316        // bind HTTP listen port317        if (port == 0) {318            int bound_port = srv->bind_to_any_port(hostname);319            was_bound = (bound_port >= 0);320            if (was_bound) {321                port = bound_port;322            }323        } else {324            was_bound = srv->bind_to_port(hostname, port);325        }326    }327 328    if (!was_bound) {329        LOG_ERR("%s: couldn't bind HTTP server socket, hostname: %s, port: %d\n", __func__, hostname.c_str(), port);330        return false;331    }332 333    // run the HTTP server in a thread334    thread = std::thread([this]() { pimpl->srv->listen_after_bind(); });335    srv->wait_until_ready();336 337    listening_address = is_sock ? string_format("unix://%s",    hostname.c_str())338                                : string_format("http://%s:%d", hostname.c_str(), port);339    return true;340}341 342void server_http_context::stop() const {343    if (pimpl->srv) {344        pimpl->srv->stop();345    }346}347 348static void set_headers(httplib::Response & res, const std::map<std::string, std::string> & headers) {349    for (const auto & [key, value] : headers) {350        res.set_header(key, value);351    }352}353 354static std::map<std::string, std::string> get_params(const httplib::Request & req) {355    std::map<std::string, std::string> params;356    for (const auto & [key, value] : req.params) {357        params[key] = value;358    }359    for (const auto & [key, value] : req.path_params) {360        params[key] = value;361    }362    return params;363}364 365static std::map<std::string, std::string> get_headers(const httplib::Request & req) {366    std::map<std::string, std::string> headers;367    for (const auto & [key, value] : req.headers) {368        headers[key] = value;369    }370    return headers;371}372 373static std::string build_query_string(const httplib::Request & req) {374    std::string qs;375    for (const auto & [key, value] : req.params) {376        if (!qs.empty()) {377            qs += '&';378        }379        qs += httplib::encode_query_component(key) + "=" + httplib::encode_query_component(value);380    }381    return qs;382}383 384// using unique_ptr for request to allow safe capturing in lambdas385using server_http_req_ptr = std::unique_ptr<server_http_req>;386 387static void process_handler_response(server_http_req_ptr && request, server_http_res_ptr & response, httplib::Response & res) {388    if (response->is_stream()) {389        res.status = response->status;390        set_headers(res, response->headers);391        std::string content_type = response->content_type;392        // convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it393        std::shared_ptr<server_http_req> q_ptr = std::move(request);394        std::shared_ptr<server_http_res> r_ptr = std::move(response);395        const auto chunked_content_provider = [response = r_ptr](size_t, httplib::DataSink & sink) -> bool {396            std::string chunk;397            bool has_next = response->next(chunk);398            if (!chunk.empty()) {399                if (!sink.write(chunk.data(), chunk.size())) {400                    return false;401                }402                SRV_DBG("http: streamed chunk: %s\n", chunk.c_str());403            }404            if (!has_next) {405                sink.done();406                SRV_DBG("%s", "http: stream ended\n");407            }408            return has_next;409        };410        const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable {411            response.reset(); // trigger the destruction of the response object412            request.reset();  // trigger the destruction of the request object413        };414        res.set_chunked_content_provider(content_type, chunked_content_provider, on_complete);415    } else {416        res.status = response->status;417        set_headers(res, response->headers);418        res.set_content(response->data, response->content_type);419    }420}421 422void server_http_context::get(const std::string & path, const server_http_context::handler_t & handler) const {423    pimpl->srv->Get(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {424        server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{425            get_params(req),426            get_headers(req),427            req.path,428            build_query_string(req),429            req.body,430            {},431            req.is_connection_closed432        });433        server_http_res_ptr response = handler(*request);434        process_handler_response(std::move(request), response, res);435    });436}437 438void server_http_context::post(const std::string & path, const server_http_context::handler_t & handler) const {439    pimpl->srv->Post(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {440        std::string body = req.body;441        std::map<std::string, raw_buffer> files;442 443        if (req.is_multipart_form_data()) {444            // translate text fields to a JSON object and use it as the body445            json form_json = json::object();446            for (const auto & [key, field] : req.form.fields) {447                if (form_json.contains(key)) {448                    // if the key already exists, convert it to an array449                    if (!form_json[key].is_array()) {450                        json existing_value = form_json[key];451                        form_json[key] = json::array({existing_value});452                    }453                    form_json[key].push_back(field.content);454                } else {455                    form_json[key] = field.content;456                }457            }458            body = form_json.dump();459 460            // populate files from multipart form461            for (const auto & [key, file] : req.form.files) {462                files[key] = raw_buffer(file.content.begin(), file.content.end());463            }464        }465 466        server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{467            get_params(req),468            get_headers(req),469            req.path,470            build_query_string(req),471            body,472            std::move(files),473            req.is_connection_closed474        });475        server_http_res_ptr response = handler(*request);476        process_handler_response(std::move(request), response, res);477    });478}479 480