Felipe97/llama-cpp-compiled

1#pragma once2 3#include "server-http.h"4#include "server-task.h"5#include "server-queue.h"6 7#include "json.h"8 9#include <cstddef>10#include <memory>11#include <mutex>12#include <set>13 14struct server_context_impl; // private implementation15 16struct server_context_meta {17    std::string build_info;18    std::string model_name;19    std::set<std::string> model_aliases;20    std::set<std::string> model_tags;21    std::string model_path;22    bool has_mtmd;23    bool has_inp_image;24    bool has_inp_audio;25    bool has_inp_video;26    json json_ui_settings;27    int slot_n_ctx;28    enum llama_pooling_type pooling_type;29 30    // chat params31    server_chat_params & chat_params;32    std::map<std::string, bool> chat_template_caps;33 34    // tokens35    std::string bos_token_str;36    std::string eos_token_str;37    llama_token fim_pre_token;38    llama_token fim_sub_token;39    llama_token fim_mid_token;40    llama_token fim_pad_token;41    llama_token fim_rep_token;42    llama_token fim_sep_token;43 44    // sampling45    std::vector<llama_logit_bias> logit_bias_eog;46 47    // model meta48    enum llama_vocab_type model_vocab_type;49    int32_t model_vocab_n_tokens;50    int32_t model_n_ctx_train;51    int32_t model_n_embd_inp;52    uint64_t model_n_params;53    uint64_t model_size;54    std::string model_ftype;55};56 57enum server_state {58    SERVER_STATE_DOWNLOADING,59    SERVER_STATE_LOADING,60    SERVER_STATE_READY,61    SERVER_STATE_SLEEPING,62};63 64static std::string server_state_to_str(server_state state) {65    switch (state) {66        case SERVER_STATE_DOWNLOADING: return "downloading";67        case SERVER_STATE_LOADING:     return "loading";68        case SERVER_STATE_READY:       return "ready";69        case SERVER_STATE_SLEEPING:    return "sleeping";70        default: GGML_ASSERT(false && "invalid server_state");71    }72}73 74static server_state server_state_from_str(const std::string & str) {75    if (str == "downloading") return SERVER_STATE_DOWNLOADING;76    if (str == "loading")     return SERVER_STATE_LOADING;77    if (str == "ready")       return SERVER_STATE_READY;78    if (str == "sleeping")    return SERVER_STATE_SLEEPING;79    GGML_ASSERT(false && "invalid server_state string");80}81 82using server_state_callback_t = std::function<void(server_state, json /* payload */)>;83 84struct server_context {85    std::unique_ptr<server_context_impl> impl;86 87    server_context();88    ~server_context();89 90    // load the model and initialize llama_context91    // returns true on success92    bool load_model(common_params & params);93 94    // this function will block main thread until termination95    void start_loop();96 97    // terminate main loop (will unblock start_loop)98    void terminate();99 100    // get the underlaying llama_context, can return nullptr if sleeping101    // not thread-safe, should only be used from the main thread102    llama_context * get_llama_context() const;103 104    // get a new response reader, used by CLI application105    server_response_reader get_response_reader();106 107    // get server metadata (read-only), can only be called after load_model()108    // not thread-safe, should only be used from the main thread109    server_context_meta get_meta() const;110 111    // note: must be set before load_model() is called112    void set_state_callback(server_state_callback_t callback);113};114 115 116// forward declarations117struct server_res_generator;118 119struct server_routes {120    server_routes(const common_params & params, server_context & ctx_server);121 122    void init_routes();123 124    // note: this is not thread-safe and can only when ctx_http.is_ready is false125    void update_meta(const server_context & ctx_server) {126        this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());127    }128 129    // handlers using lambda function, so that they can capture `this` without `std::bind`130    // they won't be called until ctx_http.is_ready is set to true131    server_http_context::handler_t get_health;132    server_http_context::handler_t get_metrics;133    server_http_context::handler_t get_slots;134    server_http_context::handler_t post_slots;135    server_http_context::handler_t get_props;136    server_http_context::handler_t post_props;137    server_http_context::handler_t post_infill;138    server_http_context::handler_t post_completions;139    server_http_context::handler_t post_completions_oai;140    server_http_context::handler_t post_chat_completions;141    server_http_context::handler_t post_chat_completions_tok;142    server_http_context::handler_t post_control;143    server_http_context::handler_t post_responses_oai;144    server_http_context::handler_t post_responses_tok_oai;145    server_http_context::handler_t post_transcriptions_oai;146    server_http_context::handler_t post_anthropic_messages;147    server_http_context::handler_t post_anthropic_count_tokens;148    server_http_context::handler_t post_apply_template;149    server_http_context::handler_t get_models;150    server_http_context::handler_t post_tokenize;151    server_http_context::handler_t post_detokenize;152    server_http_context::handler_t post_embeddings;153    server_http_context::handler_t post_embeddings_oai;154    server_http_context::handler_t post_rerank;155    server_http_context::handler_t get_lora_adapters;156    server_http_context::handler_t post_lora_adapters;157 158    // to be used in router mode159    json get_model_info() const;160 161private:162    std::unique_ptr<server_res_generator> handle_completions_impl(163            const server_http_req & req,164            server_task_type type,165            const json & data,166            const std::vector<raw_buffer> & files,167            task_response_type res_type);168    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);169    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);170    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);171    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);172    std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type);173 174    // using unique_ptr to allow late initialization of const175    std::unique_ptr<const server_context_meta> meta;176 177    const common_params & params;178    server_context_impl & ctx_server;179 180    server_queue & queue_tasks;181    server_response & queue_results;182    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);183 184    // cached responses, to be used during sleep185    std::mutex     mutex_cache;186    json           cached_models  = nullptr;187    json           cached_props   = nullptr;188    server_metrics cached_metrics;189    // set when a scrape during sleep already reported the throughput buckets190    bool           should_reset_buckets = false;191    // call right before sleep to update the cached responses192    void update_cached_responses(bool is_sleeping);193};194