Felipe97/llama-cpp-compiled

1#pragma once2 3#include <thread>4 5#include "http.h"6 7// llama_server will be available as a dynamic library symbol8int llama_server(common_params & params, int argc, char ** argv);9void llama_server_terminate();10 11struct cli_server {12    std::thread th;13    int port = -1;14    std::atomic<bool> is_alive = false;15    std::atomic<bool> is_stopping = false;16 17    ~cli_server() {18        stop();19    }20 21    void stop() {22        if (is_stopping.exchange(true)) {23            return;24        }25        if (alive()) {26            llama_server_terminate();27        }28        if (th.joinable()) {29            th.join();30        }31    }32 33    // spawn llama-server in a thread and interact with it via a random port34    bool start(common_params & params) {35        port = common_http_get_free_port();36        if (port <= 0) {37            fprintf(stderr, "failed to get a free port\n");38            exit(1);39        }40 41        is_alive.store(true, std::memory_order_release);42 43        common_params server_params = params; // copy44        server_params.port = port;45 46        th = std::thread([this, server_params]() mutable {47            // argc / argv are only used in router mode, we can skip them for now48            int res = llama_server(server_params, 0, nullptr);49            if (res != 0) {50                fprintf(stderr, "llama_server exited with code %d\n", res);51            }52            is_alive.store(false, std::memory_order_release);53        });54 55        return true;56    }57 58    std::string address() const {59        return "http://127.0.0.1:" + std::to_string(port);60    }61 62    bool wait_ready(std::function<bool()> should_stop) {63        if (!alive()) {64            return false;65        }66        while (!should_stop()) {67            auto [cli, parts] = common_http_client(address());68            cli.set_connection_timeout(1, 0);69            auto res = cli.Get("/health");70            if (res) {71                if (res->status == 200) {72                    return true;73                }74                // any other status means the server is up but not ready yet75                // (e.g. 503 while the model is still loading)76            }77            if (!alive()) {78                // in case server die permanently79                return false;80            }81            std::this_thread::sleep_for(std::chrono::milliseconds(200));82        }83        return true;84    }85 86    bool alive() const {87        return is_alive.load(std::memory_order_acquire);88    }89};90