Felipe97/llama-cpp-compiled
01.1k
1#include "common.h"2#include "http.h"3#include "server-http.h"4#include "server-common.h"5#include "ui.h"6 7#include <cpp-httplib/httplib.h>8 9#include <functional>10#include <future>11#include <memory>12#include <string>13#include <thread>14 15//16// HTTP implementation using cpp-httplib17//18 19class server_http_context::Impl {20public:21 std::unique_ptr<httplib::Server> srv;22};23 24server_http_context::server_http_context()25 : pimpl(std::make_unique<Impl>())26{}27 28server_http_context::~server_http_context() = default;29 30static void log_server_request(const httplib::Request & req, const httplib::Response & res) {31 // skip logging requests that are regularly sent, to avoid log spam32 if (req.path == "/health"33 || req.path == "/v1/health"34 || req.path == "/models"35 || req.path == "/v1/models"36 || req.path == "/props"37 || req.path == "/metrics"38 ) {39 return;40 }41 42 // reminder: this function is not covered by httplib's exception handler; if someone does more complicated stuff, think about wrapping it in try-catch43 44 SRV_TRC("done request: %s %s %s %d\n", req.method.c_str(), req.path.c_str(), req.remote_addr.c_str(), res.status);45 46 SRV_DBG("request: %s\n", req.body.c_str());47 SRV_DBG("response: %s\n", res.body.c_str());48}49 50// returns true if the Origin header value's host is localhost / 127.0.0.1 / ::1 (any port)51static bool origin_is_localhost(const std::string & origin) {52 try {53 const std::string host = common_http_parse_url(origin).host;54 return host == "localhost" || host == "127.0.0.1" || host == "::1";55 } catch (const std::exception &) {56 return false;57 }58}59 60// For Google Cloud Platform deployment compatibility61struct gcp_params {62 bool enabled;63 std::string path_health;64 std::string path_predict;65 int port;66 67 // Ref: https://docs.cloud.google.com/vertex-ai/docs/predictions/custom-container-requirements#aip-variables68 gcp_params() {69 enabled = getenv("AIP_MODE", "") == "PREDICTION";70 path_health = getenv("AIP_HEALTH_ROUTE", "", true); // default: using the route defined in server.cpp71 path_predict = getenv("AIP_PREDICT_ROUTE", "/predict", true);72 port = std::stoi(getenv("AIP_HTTP_PORT", "8080"));73 }74 75 static std::string getenv(const char * name, const std::string & default_value, bool ensure_leading_slash = false) {76 const auto * value = std::getenv(name);77 if (value == nullptr || value[0] == '\0') {78 return default_value;79 }80 std::string val = value;81 if (ensure_leading_slash && !val.empty() && val[0] != '/') {82 val.insert(val.begin(), '/');83 }84 return val;85 }86};87 88bool server_http_context::init(const common_params & params) {89 const gcp_params gcp;90 91 path_prefix = params.api_prefix;92 port = params.port;93 hostname = params.hostname;94 95 if (gcp.enabled) {96 SRV_TRC("Google Cloud Platform compat: health route = %s, predict route = %s, port = %d\n", gcp.path_health.c_str(), gcp.path_predict.c_str(), gcp.port);97 98 if (port != gcp.port) {99 SRV_WRN("Google Cloud Platform compat: overriding server port %d with AIP_HTTP_PORT %d\n", port, gcp.port);100 }101 102 port = gcp.port;103 }104 105 auto & srv = pimpl->srv;106 107#ifdef CPPHTTPLIB_OPENSSL_SUPPORT108 if (!params.ssl_file_key.empty() && !params.ssl_file_cert.empty()) {109 SRV_TRC("running with SSL: key = %s, cert = %s\n", params.ssl_file_key.c_str(), params.ssl_file_cert.c_str());110 srv = std::make_unique<httplib::SSLServer>(111 params.ssl_file_cert.c_str(), params.ssl_file_key.c_str()112 );113 is_ssl = true;114 } else {115 SRV_TRC("%s", "running without SSL\n");116 srv = std::make_unique<httplib::Server>();117 }118#else119 if (params.ssl_file_key != "" && params.ssl_file_cert != "") {120 SRV_ERR("%s", "the server is built without SSL support\n");121 return false;122 }123 srv.reset(new httplib::Server());124#endif125 126 srv->set_default_headers({{"Server", "llama.cpp"}});127 // srv->set_logger(log_server_request); // TODO @ngxson : this is too spamy, no very useful; improve it in the future128 srv->set_exception_handler([](const httplib::Request &, httplib::Response & res, const std::exception_ptr & ep) {129 // this is fail-safe; exceptions should already handled by `ex_wrapper`130 131 std::string message;132 try {133 std::rethrow_exception(ep);134 } catch (const std::exception & e) {135 message = e.what();136 } catch (...) {137 message = "Unknown Exception";138 }139 140 res.status = 500;141 res.set_content(message, "text/plain");142 SRV_ERR("got exception: %s\n", message.c_str());143 });144 145 srv->set_error_handler([](const httplib::Request &, httplib::Response & res) {146 if (res.status == 404) {147 res.set_content(148 safe_json_to_str(json {149 {"error", {150 {"message", "File Not Found"},151 {"type", "not_found_error"},152 {"code", 404}153 }}154 }),155 "application/json; charset=utf-8"156 );157 }158 // for other error codes, we skip processing here because it's already done by res->error()159 });160 161 // set timeouts and change hostname and port162 srv->set_read_timeout (params.timeout_read);163 srv->set_write_timeout(params.timeout_write);164 srv->set_socket_options([reuse_port = params.reuse_port](const socket_t sock) {165 httplib::set_socket_opt(sock, SOL_SOCKET, SO_REUSEADDR, 1);166 if (reuse_port) {167#ifdef SO_REUSEPORT168 httplib::set_socket_opt(sock, SOL_SOCKET, SO_REUSEPORT, 1);169#else170 SRV_WRN("%s", "SO_REUSEPORT is not supported\n");171#endif172 }173 });174 175 if (params.api_keys.size() == 1) {176 const auto key = params.api_keys[0];177 const std::string substr = key.substr(std::max(static_cast<int>(key.length() - 4), 0));178 SRV_TRC("api_keys: ****%s\n", substr.c_str());179 } else if (params.api_keys.size() > 1) {180 SRV_TRC("api_keys: %zu keys loaded\n", params.api_keys.size());181 }182 183 //184 // Middlewares185 //186 187 // Frontend paths - all embedded UI assets188 static const std::unordered_set<std::string> frontend_paths = []() {189 std::unordered_set<std::string> paths { "/" };190 for (const llama_ui_asset & a : llama_ui_get_assets()) {191 paths.insert("/" + a.name);192 }193 return paths;194 }();195 196 // Public endpoints - API routes plus all embedded UI assets197 static const std::unordered_set<std::string> get_public_endpoints = []() {198 std::unordered_set<std::string> endpoints {199 "/health",200 "/v1/health",201 };202 endpoints.insert(frontend_paths.begin(), frontend_paths.end());203 return endpoints;204 }();205 206 auto middleware_validate_api_key = [api_keys = params.api_keys](const httplib::Request & req, httplib::Response & res) {207 // If API key is not set, skip validation208 if (api_keys.empty()) {209 return true;210 }211 212 // If path is public or a UI asset, skip validation213 if (get_public_endpoints.count(req.path)) {214 return true;215 }216 217 // Check for API key in the Authorization header218 std::string req_api_key = req.get_header_value("Authorization");219 if (req_api_key.empty()) {220 // retry with anthropic header221 req_api_key = req.get_header_value("X-Api-Key");222 }223 224 // remove the "Bearer " prefix if needed225 static std::string prefix = "Bearer ";226 if (req_api_key.substr(0, prefix.size()) == prefix) {227 req_api_key = req_api_key.substr(prefix.size());228 }229 230 // validate the API key231 if (std::find(api_keys.begin(), api_keys.end(), req_api_key) != api_keys.end()) {232 return true; // API key is valid233 }234 235 // API key is invalid or not provided236 res.status = 401;237 res.set_content(238 safe_json_to_str(json {239 {"error", {240 {"message", "Invalid API Key"},241 {"type", "authentication_error"},242 {"code", 401}243 }}244 }),245 "application/json; charset=utf-8"246 );247 248 SRV_WRN("%s", "unauthorized: Invalid API Key\n");249 250 return false;251 };252 253 auto middleware_server_state = [this](const httplib::Request & req, httplib::Response & res) {254 if (!is_ready.load()) {255 if (frontend_paths.count(req.path)) {256 return true; // frontend asset, allow it to load and show "loading"257 }258 // no endpoints are allowed to be accessed when the server is not ready259 // this is to prevent any data races or inconsistent states260 res.status = 503;261 res.set_content(262 safe_json_to_str(json {263 {"error", {264 {"message", "Loading model"},265 {"type", "unavailable_error"},266 {"code", 503}267 }}268 }),269 "application/json; charset=utf-8"270 );271 return false;272 }273 return true;274 };275 276 // register server middlewares277 srv->set_pre_routing_handler([¶ms, middleware_validate_api_key, middleware_server_state](const httplib::Request & req, httplib::Response & res) {278 if (params.cors_credentials && params.cors_origins == "*") {279 // special case: echo back the Origin header to allow any origin to access the server with credentials280 res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));281 } else if (params.cors_origins == "localhost") {282 // special case: only reflect the Origin header if it is a localhost origin283 std::string origin = req.get_header_value("Origin");284 if (!origin.empty() && origin_is_localhost(origin)) {285 res.set_header("Access-Control-Allow-Origin", origin);286 } else if (!origin.empty()) {287 SRV_WRN("(CORS) skip non-localhost origin: %s\n", origin.c_str());288 }289 } else {290 res.set_header("Access-Control-Allow-Origin", params.cors_origins);291 }292 // If this is OPTIONS request, skip validation because browsers don't include Authorization header293 if (req.method == "OPTIONS") {294 res.set_header("Access-Control-Allow-Credentials", params.cors_credentials ? "true" : "false");295 res.set_header("Access-Control-Allow-Methods", params.cors_methods);296 res.set_header("Access-Control-Allow-Headers", params.cors_headers);297 res.set_content("", "text/html"); // blank response, no data298 return httplib::Server::HandlerResponse::Handled; // skip further processing299 }300 if (!middleware_server_state(req, res)) {301 return httplib::Server::HandlerResponse::Handled;302 }303 if (!middleware_validate_api_key(req, res)) {304 return httplib::Server::HandlerResponse::Handled;305 }306 return httplib::Server::HandlerResponse::Unhandled;307 });308 309 auto n_threads_http = params.n_threads_http;310 if (n_threads_http < 1) {311 // +4 threads for monitoring, health and some threads reserved for MCP and other tasks in the future312 n_threads_http = std::max(params.n_parallel + 4, static_cast<int32_t>(std::thread::hardware_concurrency() - 1));313 }314 SRV_TRC("using %d threads for HTTP server\n", n_threads_http);315 srv->new_task_queue = [n_threads_http] {316 // spawn n_threads_http fixed thread (always alive), while allow up to 1024 max possible additional threads317 // when n_threads_http is used, server will create new "dynamic" threads that will be destroyed after processing each request318 // ref: https://github.com/yhirose/cpp-httplib/pull/2368319 const auto max_threads = static_cast<size_t>(n_threads_http + 1024);320 return new httplib::ThreadPool(n_threads_http, max_threads);321 };322 323 //324 // Web UI setup325 //326 327 // Use new `params.ui` field (backed by old `params.webui` for compat)328 if (!params.ui) {329 SRV_INF("%s", "The UI is disabled\n");330 SRV_INF("%s", "Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable\n");331 } else {332 // register static assets routes333 if (!params.public_path.empty()) {334 // Set the base directory for serving static files335 if (const auto is_found = srv->set_mount_point(params.api_prefix + "/", params.public_path); !is_found) {336 SRV_ERR("static assets path not found: %s\n", params.public_path.c_str());337 return false;338 }339 } else {340#if defined(LLAMA_UI_HAS_ASSETS)341 static auto handle_gzip_header = [](const httplib::Request & req, httplib::Response & res) {342 if (!llama_ui_use_gzip()) {343 // no gzip build, skip344 return true;345 }346 if (req.get_header_value("Accept-Encoding").find("gzip") == std::string::npos) {347 res.status = 415; // unsupported media type348 res.set_content("Error: gzip is not supported by this browser", "text/plain");349 return false;350 } else {351 res.set_header("Content-Encoding", "gzip");352 }353 return true;354 };355 356 // Hashed assets never change under a given name, so they can be cached forever.357 // `index.html` is the exception: its name is stable while its contents change on358 // every build, and it is what names the hashed asset versions the UI loads.359 static constexpr auto cache_immutable = "public, max-age=31536000, immutable";360 static constexpr auto cache_revalidate = "no-cache";361 362 // Serves an asset with ETag/304 handling, under the given caching policy.363 auto serve_asset_cached = [](const std::string & name, bool isolation, const char * cache_control) {364 return [name, isolation, cache_control](const httplib::Request & req, httplib::Response & res) {365 if (!handle_gzip_header(req, res)) {366 return true; // returns error message367 }368 const llama_ui_asset * a = llama_ui_find_asset(name);369 if (!a) { res.status = 404; return false; }370 res.set_header("ETag", a->etag);371 if (const std::string & inm = req.get_header_value("If-None-Match");372 !inm.empty() && (inm == a->etag || inm == std::string("W/") + a->etag)) {373 res.status = 304;374 return false;375 }376 if (isolation) {377 res.set_header("Cross-Origin-Embedder-Policy", "require-corp");378 res.set_header("Cross-Origin-Opener-Policy", "same-origin");379 }380 res.set_header("Cache-Control", cache_control);381 res.set_content(reinterpret_cast<const char*>(a->data), a->size, a->type.c_str());382 return false;383 };384 };385 386 auto serve_asset_nocache = [](const std::string & name) {387 return [name](const httplib::Request & req, httplib::Response & res) {388 if (!handle_gzip_header(req, res)) {389 return true; // returns error message390 }391 const llama_ui_asset * a = llama_ui_find_asset(name);392 if (!a) {393 res.status = 404;394 return false;395 }396 res.set_header("Cache-Control", "no-cache");397 res.set_content(reinterpret_cast<const char*>(a->data), a->size, a->type.c_str());398 return false;399 };400 };401 402 // main index file -- revalidated, so a new build is picked up on the next load403 srv->Get(params.api_prefix + "/", serve_asset_cached("index.html", true, cache_revalidate));404 srv->Get(params.api_prefix + "/index.html", serve_asset_cached("index.html", true, cache_revalidate));405 406 // All remaining assets registered directly from the embedded asset table.407 // PWA revalidation files (sw.js, manifest, version.json) use no-cache;408 // everything else is immutable.409 static const std::unordered_set<std::string> no_cache_names = {410 "sw.js",411 "manifest.webmanifest",412 "_app/version.json",413 "build.json"414 };415 416 for (const auto & a : llama_ui_get_assets()) {417 if (a.name == "index.html") continue; // served at "/" and "/index.html" above418 if (no_cache_names.count(a.name)) {419 SRV_DBG("serve nocache for %s\n", a.name.c_str());420 srv->Get(params.api_prefix + "/" + a.name, serve_asset_nocache(a.name));421 } else {422 srv->Get(params.api_prefix + "/" + a.name, serve_asset_cached(a.name, false, cache_immutable));423 }424 }425 426#endif427 }428 }429 return true;430}431 432bool server_http_context::start() {433 // Bind and listen434 435 const auto & srv = pimpl->srv;436 auto was_bound = false;437 auto is_sock = false;438 if (string_ends_with(std::string(hostname), ".sock")) {439 is_sock = true;440 SRV_TRC("%s", "setting address family to AF_UNIX\n");441 srv->set_address_family(AF_UNIX);442 // bind_to_port requires a second arg, any value other than 0 should443 // simply get ignored444 was_bound = srv->bind_to_port(hostname, 8080);445 } else {446 SRV_TRC("%s", "binding port with default address family\n");447 // bind HTTP listen port448 if (port == 0) {449 const auto bound_port = srv->bind_to_any_port(hostname);450 was_bound = (bound_port >= 0);451 if (was_bound) {452 port = bound_port;453 }454 } else {455 was_bound = srv->bind_to_port(hostname, port);456 }457 }458 459 if (!was_bound) {460 SRV_ERR("couldn't bind HTTP server socket, hostname: %s, port: %d\n", hostname.c_str(), port);461 return false;462 }463 464 // run the HTTP server in a thread465 thread = std::thread([this] { pimpl->srv->listen_after_bind(); });466 srv->wait_until_ready();467 468 listening_address = is_sock ? string_format("unix://%s", hostname.c_str())469 : string_format("%s://%s:%d", is_ssl ? "https" : "http", common_http_format_host(hostname).c_str(), port);470 return true;471}472 473void server_http_context::stop() const {474 if (pimpl->srv) {475 pimpl->srv->stop();476 }477}478 479static void set_headers(httplib::Response & res, const std::map<std::string, std::string> & headers) {480 for (const auto & [key, value] : headers) {481 res.set_header(key, value);482 }483}484 485// percent-decode a path component (%XX). path params arrive raw from httplib, unlike query486// params, so a conv id like "conv::model" sent as "conv%3A%3Amodel" must be decoded here to487// match the value the client put in the X-Conversation-Id header488static std::string decode_path_component(const std::string & in) {489 std::string out;490 out.reserve(in.size());491 for (size_t i = 0; i < in.size(); i++) {492 if (in[i] == '%' && i + 2 < in.size()) {493 auto hex = [](char c) -> int {494 if (c >= '0' && c <= '9') return c - '0';495 if (c >= 'a' && c <= 'f') return c - 'a' + 10;496 if (c >= 'A' && c <= 'F') return c - 'A' + 10;497 return -1;498 };499 int hi = hex(in[i + 1]);500 int lo = hex(in[i + 2]);501 if (hi >= 0 && lo >= 0) {502 out.push_back(char((hi << 4) | lo));503 i += 2;504 continue;505 }506 }507 out.push_back(in[i]);508 }509 return out;510}511 512static std::map<std::string, std::string> get_params(const httplib::Request & req) {513 std::map<std::string, std::string> params;514 for (const auto & [key, value] : req.params) {515 params[key] = value;516 }517 for (const auto & [key, value] : req.path_params) {518 params[key] = decode_path_component(value);519 }520 return params;521}522 523static std::map<std::string, std::string> get_headers(const httplib::Request & req) {524 std::map<std::string, std::string> headers;525 for (const auto & [key, value] : req.headers) {526 headers[key] = value;527 }528 return headers;529}530 531static std::string build_query_string(const httplib::Request & req) {532 std::string qs;533 for (const auto & [key, value] : req.params) {534 if (!qs.empty()) {535 qs += '&';536 }537 qs += httplib::encode_query_component(key) + "=" + httplib::encode_query_component(value);538 }539 return qs;540}541 542// using unique_ptr for request to allow safe capturing in lambdas543using server_http_req_ptr = std::unique_ptr<server_http_req>;544 545static void process_handler_response(server_http_req_ptr && request, server_http_res_ptr & response, httplib::Response & res) {546 if (response->is_stream()) {547 res.status = response->status;548 // Tell Nginx to not buffer any streamed response549 response->headers["X-Accel-Buffering"] = "no";550 set_headers(res, response->headers);551 const std::string content_type = response->content_type;552 // convert to shared_ptr as both chunked_content_provider() and on_complete() need to use it553 std::shared_ptr<server_http_req> q_ptr = std::move(request);554 std::shared_ptr<server_http_res> r_ptr = std::move(response);555 556 const auto chunked_content_provider = [response = r_ptr](size_t, httplib::DataSink & sink) -> bool {557 std::string chunk;558 const bool has_next = response->next(chunk);559 if (!chunk.empty()) {560 if (!sink.write(chunk.data(), chunk.size())) {561 return false;562 }563 SRV_DBG("http: streamed chunk: %s\n", chunk.c_str());564 }565 if (!has_next) {566 sink.done();567 SRV_DBG("%s", "http: stream ended\n");568 }569 return has_next;570 };571 const auto on_complete = [request = q_ptr, response = r_ptr](bool) mutable {572 response->on_complete();573 response.reset();574 request.reset();575 };576 res.set_chunked_content_provider(content_type, chunked_content_provider, on_complete);577 } else {578 res.status = response->status;579 set_headers(res, response->headers);580 res.set_content(response->data, response->content_type);581 response->on_complete();582 }583}584 585void server_http_context::get(const std::string & path, const server_http_context::handler_t & handler) const {586 handlers.emplace(path, handler);587 pimpl->srv->Get(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {588 server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{589 get_params(req),590 get_headers(req),591 req.path,592 build_query_string(req),593 req.body,594 {},595 req.is_connection_closed596 });597 server_http_res_ptr response = handler(*request);598 process_handler_response(std::move(request), response, res);599 });600}601 602void server_http_context::post(const std::string & path, const server_http_context::handler_t & handler) const {603 handlers.emplace(path, handler);604 pimpl->srv->Post(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {605 std::string body = req.body;606 std::map<std::string, uploaded_file> files;607 608 if (req.is_multipart_form_data()) {609 // translate text fields to a JSON object and use it as the body610 json form_json = json::object();611 for (const auto & [key, field] : req.form.fields) {612 if (form_json.contains(key)) {613 // if the key already exists, convert it to an array614 if (!form_json[key].is_array()) {615 json existing_value = form_json[key];616 form_json[key] = json::array({existing_value});617 }618 form_json[key].push_back(field.content);619 } else {620 form_json[key] = field.content;621 }622 }623 body = form_json.dump();624 625 // populate files from multipart form626 for (const auto & [key, file] : req.form.files) {627 files[key] = uploaded_file{628 raw_buffer(file.content.begin(), file.content.end()),629 file.filename,630 file.content_type,631 };632 }633 }634 635 server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{636 get_params(req),637 get_headers(req),638 req.path,639 build_query_string(req),640 body,641 std::move(files),642 req.is_connection_closed643 });644 server_http_res_ptr response = handler(*request);645 process_handler_response(std::move(request), response, res);646 });647}648 649void server_http_context::del(const std::string & path, const server_http_context::handler_t & handler) const {650 handlers.emplace(path, handler);651 pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) {652 server_http_req_ptr request = std::make_unique<server_http_req>(server_http_req{653 get_params(req),654 get_headers(req),655 req.path,656 build_query_string(req),657 req.body,658 {},659 req.is_connection_closed660 });661 server_http_res_ptr response = handler(*request);662 process_handler_response(std::move(request), response, res);663 });664}665 666//667// Vertex AI Prediction protocol (AIP_PREDICT_ROUTE)668// https://cloud.google.com/vertex-ai/docs/predictions/custom-container-requirements669//670 671// Derives the camelCase @requestFormat alias for a registered path.672// e.g. "/v1/chat/completions" -> "chatCompletions", "/apply-template" -> "applyTemplate"673static std::string path_to_gcp_format(const std::string & path) {674 std::string s = path;675 if (s.size() > 3 && s[0] == '/' && s[1] == 'v' && s[2] == '1') {676 s = s.substr(3);677 }678 if (!s.empty() && s[0] == '/') {679 s = s.substr(1);680 }681 std::string result;682 bool cap = false;683 for (unsigned char c : s) {684 if (c == ':') break; // stop before path parameters685 if (c == '/' || c == '-' || c == '_') {686 cap = true;687 } else {688 result += static_cast<char>(cap ? std::toupper(c) : c);689 cap = false;690 }691 }692 return result;693}694 695static json parse_gcp_predict_response(const server_http_res_ptr & res) {696 if (res == nullptr) {697 throw std::runtime_error("empty response from internal handler");698 }699 if (res->is_stream()) {700 throw std::invalid_argument("predict route does not support streaming responses");701 }702 if (res->data.empty()) {703 return nullptr;704 }705 try {706 return json::parse(res->data);707 } catch (...) {708 return res->data;709 }710}711 712void server_http_context::register_gcp_compat() const {713 const gcp_params gcp;714 715 if (!gcp.enabled) {716 // do nothing717 return;718 }719 720 if (handlers.count(gcp.path_predict)) {721 SRV_ERR("AIP_PREDICT_ROUTE=%s conflicts with an existing llama-server route\n", gcp.path_predict.c_str());722 exit(1);723 }724 725 // camelCase alias -> canonical path (first registration wins on collision)726 // e.g. "chatCompletions" -> "/v1/chat/completions"727 std::unordered_map<std::string, std::string> alias_to_path;728 for (const auto & [path, _] : handlers) {729 alias_to_path.emplace(path_to_gcp_format(path), path);730 }731 732 if (!gcp.path_health.empty()) {733 const auto health_handler = handlers.find("/health");734 GGML_ASSERT(health_handler != handlers.end());735 get(gcp.path_health, health_handler->second);736 }737 738 post(gcp.path_predict, [this, alias_to_path = std::move(alias_to_path)](const server_http_req & req) -> server_http_res_ptr {739 static const auto build_error = [](const std::string & message, error_type type) -> json {740 return json {{"error", format_error_response(message, type)}};741 };742 743 json data;744 try {745 data = json::parse(req.body);746 } catch (const std::exception & e) {747 auto res = std::make_unique<server_http_res>();748 res->status = 400;749 res->data = safe_json_to_str({{"error", format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)}});750 return res;751 }752 if (!data.is_object()) {753 auto res = std::make_unique<server_http_res>();754 res->status = 400;755 res->data = safe_json_to_str({{"error", format_error_response("request body must be a JSON object", ERROR_TYPE_INVALID_REQUEST)}});756 return res;757 }758 if (!data.contains("instances") || !data.at("instances").is_array()) {759 auto res = std::make_unique<server_http_res>();760 res->status = 400;761 res->data = safe_json_to_str({{"error", format_error_response("request body must include an array field named instances", ERROR_TYPE_INVALID_REQUEST)}});762 return res;763 }764 765 const json & instances = data.at("instances");766 static const size_t MAX_INSTANCES = 128;767 if (instances.size() > MAX_INSTANCES) {768 auto res = std::make_unique<server_http_res>();769 res->status = 400;770 res->data = safe_json_to_str({{"error", format_error_response("instances array exceeds maximum size of " + std::to_string(MAX_INSTANCES), ERROR_TYPE_INVALID_REQUEST)}});771 return res;772 }773 774 std::vector<std::future<json>> futures;775 futures.reserve(instances.size());776 777 for (const auto & instance : instances) {778 futures.push_back(std::async(std::launch::async, [this, &req, &alias_to_path, instance]() -> json {779 if (!instance.is_object()) {780 return build_error("each instance must be a JSON object", ERROR_TYPE_INVALID_REQUEST);781 }782 if (!instance.contains("@requestFormat") || !instance.at("@requestFormat").is_string()) {783 return build_error("each instance must include a string @requestFormat", ERROR_TYPE_INVALID_REQUEST);784 }785 786 try {787 json payload = instance;788 const std::string format = payload.at("@requestFormat").get<std::string>();789 payload.erase("@requestFormat");790 791 if (payload.contains("stream")) {792 SRV_WRN("%s", "ignoring client-provided stream field in instance, streaming is not supported in predict route\n");793 payload["stream"] = false;794 }795 796 // accept both camelCase aliases (e.g. "chatCompletions") and direct paths797 std::string dispatch_path;798 auto it_alias = alias_to_path.find(format);799 if (it_alias != alias_to_path.end()) {800 dispatch_path = it_alias->second;801 } else if (handlers.count(format)) {802 dispatch_path = format;803 } else {804 return build_error("no handler registered for @requestFormat: " + format, ERROR_TYPE_INVALID_REQUEST);805 }806 807 const server_http_req internal_req {808 req.params,809 req.headers,810 path_prefix + dispatch_path,811 req.query_string,812 payload.dump(),813 {},814 req.should_stop,815 };816 817 server_http_res_ptr internal_res = handlers.at(dispatch_path)(internal_req);818 return parse_gcp_predict_response(internal_res);819 } catch (const std::invalid_argument & e) {820 return build_error(e.what(), ERROR_TYPE_INVALID_REQUEST);821 } catch (const std::exception & e) {822 return build_error(e.what(), ERROR_TYPE_SERVER);823 } catch (...) {824 return build_error("unknown error", ERROR_TYPE_SERVER);825 }826 }));827 }828 829 json predictions = json::array();830 for (auto & future : futures) {831 predictions.push_back(future.get());832 }833 834 auto res = std::make_unique<server_http_res>();835 res->data = safe_json_to_str({{"predictions", predictions}});836 return res;837 });838}839