echodict/llama.cpp

1#pragma once2 3#include "server-http.h"4#include "server-task.h"5#include "server-queue.h"6 7#include <nlohmann/json_fwd.hpp>8 9#include <cstddef>10#include <memory>11#include <set>12 13struct server_context_impl; // private implementation14 15struct server_context_meta {16    std::string build_info;17    std::string model_name;18    std::set<std::string> model_aliases;19    std::set<std::string> model_tags;20    std::string model_path;21    bool has_mtmd;22    bool has_inp_image;23    bool has_inp_audio;24    json json_webui_settings;25    int slot_n_ctx;26    enum llama_pooling_type pooling_type;27 28    // chat params29    server_chat_params & chat_params;30    std::map<std::string, bool> chat_template_caps;31 32    // tokens33    std::string bos_token_str;34    std::string eos_token_str;35    llama_token fim_pre_token;36    llama_token fim_sub_token;37    llama_token fim_mid_token;38    llama_token fim_pad_token;39    llama_token fim_rep_token;40    llama_token fim_sep_token;41 42    // sampling43    std::vector<llama_logit_bias> logit_bias_eog;44 45    // model meta46    enum llama_vocab_type model_vocab_type;47    int32_t model_vocab_n_tokens;48    int32_t model_n_ctx_train;49    int32_t model_n_embd_inp;50    uint64_t model_n_params;51    uint64_t model_size;52};53 54struct server_context {55    std::unique_ptr<server_context_impl> impl;56 57    server_context();58    ~server_context();59 60    // load the model and initialize llama_context61    // returns true on success62    bool load_model(common_params & params);63 64    // this function will block main thread until termination65    void start_loop();66 67    // terminate main loop (will unblock start_loop)68    void terminate();69 70    // get the underlaying llama_context, can return nullptr if sleeping71    // not thread-safe, should only be used from the main thread72    llama_context * get_llama_context() const;73 74    // get a new response reader, used by CLI application75    server_response_reader get_response_reader();76 77    // get server metadata (read-only), can only be called after load_model()78    // not thread-safe, should only be used from the main thread79    server_context_meta get_meta() const;80 81    // register a callback to be called when sleeping state changes82    // must be set before load_model() is called83    void on_sleeping_changed(std::function<void(bool)> callback);84};85 86 87// forward declarations88struct server_res_generator;89 90struct server_routes {91    server_routes(const common_params & params, server_context & ctx_server);92 93    void init_routes();94 95    // note: this is not thread-safe and can only when ctx_http.is_ready is false96    void update_meta(const server_context & ctx_server) {97        this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());98    }99 100    // handlers using lambda function, so that they can capture `this` without `std::bind`101    // they won't be called until ctx_http.is_ready is set to true102    server_http_context::handler_t get_health;103    server_http_context::handler_t get_metrics;104    server_http_context::handler_t get_slots;105    server_http_context::handler_t post_slots;106    server_http_context::handler_t get_props;107    server_http_context::handler_t post_props;108    server_http_context::handler_t post_infill;109    server_http_context::handler_t post_completions;110    server_http_context::handler_t post_completions_oai;111    server_http_context::handler_t post_chat_completions;112    server_http_context::handler_t post_responses_oai;113    server_http_context::handler_t post_transcriptions_oai;114    server_http_context::handler_t post_anthropic_messages;115    server_http_context::handler_t post_anthropic_count_tokens;116    server_http_context::handler_t post_apply_template;117    server_http_context::handler_t get_models;118    server_http_context::handler_t post_tokenize;119    server_http_context::handler_t post_detokenize;120    server_http_context::handler_t post_embeddings;121    server_http_context::handler_t post_embeddings_oai;122    server_http_context::handler_t post_rerank;123    server_http_context::handler_t get_lora_adapters;124    server_http_context::handler_t post_lora_adapters;125private:126    std::unique_ptr<server_res_generator> handle_completions_impl(127            const server_http_req & req,128            server_task_type type,129            const json & data,130            const std::vector<raw_buffer> & files,131            task_response_type res_type);132    std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);133    std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);134    std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);135    std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);136 137    // using unique_ptr to allow late initialization of const138    std::unique_ptr<const server_context_meta> meta;139 140    const common_params & params;141    const server_context_impl & ctx_server;142 143    server_queue & queue_tasks;144    server_response & queue_results;145    std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);146};147