Felipe97/llama-cpp-compiled

1#include "cli-client.h"2 3#include "http.h"4 5#include <algorithm>6#include <chrono>7#include <thread>8 9// generation can stall for a long time during prompt processing, so the10// read timeout must be generous11static constexpr time_t CLI_HTTP_READ_TIMEOUT_SEC = 3600;12 13// upper bound for the accumulated response body kept for error reporting14static constexpr size_t CLI_HTTP_MAX_ERROR_BODY = 1024 * 1024;15 16// returns the path with the base url's path prefix prepended (if any)17static std::string join_path(const common_http_url & parts, const std::string & path) {18    if (parts.path.empty() || parts.path == "/") {19        return path;20    }21    std::string prefix = parts.path;22    if (prefix.back() == '/') {23        prefix.pop_back();24    }25    return prefix + path;26}27 28std::string cli_client::get(const std::string & path) {29    auto [cli, parts] = common_http_client(server_base);30    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);31    auto path_with_model = path + (model.empty() ? "" : ("?model=" + model));32    auto res = cli.Get(join_path(parts, path_with_model));33    if (!res) {34        throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));35    }36    if (res->status < 200 || res->status >= 300) {37        throw std::runtime_error("GET " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);38    }39    return res->body;40}41 42std::string cli_client::post(const std::string & path, const std::string & body) {43    auto [cli, parts] = common_http_client(server_base);44    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);45    auto res = cli.Post(join_path(parts, path), body, "application/json");46    if (!res) {47        throw std::runtime_error("failed to connect to " + server_base + ": " + httplib::to_string(res.error()));48    }49    if (res->status < 200 || res->status >= 300) {50        throw std::runtime_error("POST " + path + " failed with status " + std::to_string(res->status) + ": " + res->body);51    }52    return res->body;53}54 55std::string cli_client::post_sse(const std::string & path,56                                  const std::string & body,57                                  const std::function<bool()> & should_stop,58                                  const std::function<void(const std::string &)> & on_data) {59    auto [cli, parts] = common_http_client(server_base);60    cli.set_read_timeout(CLI_HTTP_READ_TIMEOUT_SEC, 0);61 62    std::string pending;  // buffer for incomplete SSE lines63    std::string raw_body; // accumulated body, used only for error reporting64 65    auto receiver = [&](const char * data, size_t len) -> bool {66        if (should_stop()) {67            return false; // aborts the request68        }69        if (raw_body.size() < CLI_HTTP_MAX_ERROR_BODY) {70            raw_body.append(data, std::min(len, CLI_HTTP_MAX_ERROR_BODY - raw_body.size()));71        }72        pending.append(data, len);73        size_t pos;74        while ((pos = pending.find('\n')) != std::string::npos) {75            std::string line = pending.substr(0, pos);76            pending.erase(0, pos + 1);77            if (!line.empty() && line.back() == '\r') {78                line.pop_back();79            }80            if (line.rfind("data: ", 0) != 0) {81                continue;82            }83            std::string payload = line.substr(6);84            if (payload == "[DONE]") {85                continue;86            }87            on_data(payload);88        }89        return true;90    };91 92    httplib::Headers headers = {{"Accept", "text/event-stream"}};93    auto res = cli.Post(join_path(parts, path), headers, body, "application/json", receiver);94 95    if (!res) {96        if (res.error() == httplib::Error::Canceled && should_stop()) {97            return ""; // cancelled by the user98        }99        return "failed to connect to " + server_base + ": " + httplib::to_string(res.error());100    }101    if (res->status < 200 || res->status >= 300) {102        if (!raw_body.empty()) {103            return raw_body;104        }105        return "request failed with status " + std::to_string(res->status);106    }107    return "";108}109 110bool cli_client::wait_health(const std::function<bool()> & is_aborted) {111    int connect_attempts = 0;112    while (!is_aborted()) {113        auto [cli, parts] = common_http_client(server_base);114        cli.set_connection_timeout(1, 0);115        auto res = cli.Get(join_path(parts, "/health"));116        if (res) {117            if (res->status == 200) {118                return true;119            }120            // any other status means the server is up but not ready yet121            // (e.g. 503 while the model is still loading)122        } else if (++connect_attempts >= 10) {123            last_error = "failed to connect to " + server_base + ": " + httplib::to_string(res.error());124            return false;125        }126        std::this_thread::sleep_for(std::chrono::milliseconds(300));127    }128    last_error = "aborted while waiting for the server to become ready";129    return false;130}131