echodict/llama.cpp

1#pragma once2 3#include "common.h"4#include "llama.h"5 6#include <string>7#include <unordered_set>8#include <list>9#include <map>10 11// TODO: prevent including the whole server-common.h as we only use server_tokens12#include "server-common.h"13 14using json = nlohmann::ordered_json;15 16enum server_task_type {17    SERVER_TASK_TYPE_COMPLETION,18    SERVER_TASK_TYPE_EMBEDDING,19    SERVER_TASK_TYPE_RERANK,20    SERVER_TASK_TYPE_INFILL,21    SERVER_TASK_TYPE_CANCEL,22    SERVER_TASK_TYPE_NEXT_RESPONSE,23    SERVER_TASK_TYPE_METRICS,24    SERVER_TASK_TYPE_SLOT_SAVE,25    SERVER_TASK_TYPE_SLOT_RESTORE,26    SERVER_TASK_TYPE_SLOT_ERASE,27    SERVER_TASK_TYPE_GET_LORA,28    SERVER_TASK_TYPE_SET_LORA,29};30 31// TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common32enum task_response_type {33    TASK_RESPONSE_TYPE_NONE, // llama.cpp native format34    TASK_RESPONSE_TYPE_OAI_CHAT,35    TASK_RESPONSE_TYPE_OAI_CMPL,36    TASK_RESPONSE_TYPE_OAI_RESP,37    TASK_RESPONSE_TYPE_OAI_ASR, // transcriptions API38    TASK_RESPONSE_TYPE_OAI_EMBD,39    TASK_RESPONSE_TYPE_ANTHROPIC,40};41 42enum stop_type {43    STOP_TYPE_NONE,44    STOP_TYPE_EOS,45    STOP_TYPE_WORD,46    STOP_TYPE_LIMIT,47};48 49struct task_params {50    bool stream          = true;51    bool include_usage   = false;52    bool cache_prompt    = true; // remember the prompt to avoid reprocessing all prompt53    bool return_tokens   = false;54    bool return_progress = false;55 56    int32_t n_keep    =  0; // number of tokens to keep from initial prompt57    int32_t n_discard =  0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half58    int32_t n_predict = -1; // new tokens to predict59    int32_t n_indent  =  0; // minimum line indentation for the generated text in number of whitespace characters60    int32_t n_cmpl    =  1; // number of completions to generate from this prompt61 62    int32_t n_cache_reuse = 0; // min chunk size to attempt reusing from the cache via KV shifting (0 = disabled)63 64    int64_t t_max_prompt_ms  = -1; // TODO: implement65    int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit66 67    std::map<int, float> lora; // mapping adapter ID -> scale68 69    std::vector<std::string> antiprompt;70    std::vector<std::string> response_fields;71 72    bool timings_per_token   = false;73    bool post_sampling_probs = false;74 75    struct common_params_sampling sampling;76    struct common_params_speculative speculative;77 78    // response formatting79    bool               verbose  = false;80    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;81    std::string        oaicompat_model;82    std::string        oaicompat_cmpl_id;83 84    // per-request parameters for chat parsing85    common_chat_parser_params chat_parser_params;86 87    // Embeddings88    int32_t embd_normalize = 2; // (-1=none, 0=max absolute int16, 1=taxicab, 2=Euclidean/L2, >2=p-norm)89 90    json format_logit_bias(const std::vector<llama_logit_bias> & logit_bias) const;91    json to_json(bool only_metrics = false) const;92};93 94// struct for tracking the state of a task (e.g., for streaming)95struct task_result_state {96    // tracking diffs for partial tool calls97    std::vector<common_chat_msg_diff> diffs;98    common_chat_parser_params chat_parser_params;99    common_chat_msg chat_msg;100    std::string generated_text; // append new chunks of generated text here101    std::vector<std::string> generated_tool_call_ids;102    std::unordered_set<size_t> sent_tool_call_names;103 104    // for OpenAI Responses and Anthropic streaming API:105    // track output item / content block state across chunks106    bool thinking_block_started = false;107    bool text_block_started = false;108 109    // for OpenAI Responses streaming API110    const std::string oai_resp_id;111    const std::string oai_resp_reasoning_id;112    const std::string oai_resp_message_id;113    std::string oai_resp_fc_id; // function call ID for current args delta114 115    task_result_state(const common_chat_parser_params & chat_parser_params)116        : chat_parser_params(chat_parser_params)117        , oai_resp_id("resp_" + random_string())118        , oai_resp_reasoning_id("rs_" + random_string())119        , oai_resp_message_id("msg_" + random_string()) {}120 121    // parse partial tool calls and update the internal state122    common_chat_msg update_chat_msg(123        const std::string & text_added,124        bool is_partial,125        std::vector<common_chat_msg_diff> & diffs,126        bool filter_tool_calls = false);127};128 129struct server_task {130    int id = -1; // to be filled by server_queue131 132    // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader133    size_t index = 0; // used when there are multiple prompts (batch request)134 135    // used by SERVER_TASK_TYPE_CANCEL136    int id_target = -1;137    int id_slot   = -1;138 139    // used by parallel sampling (multiple completions from same prompt)140    int id_parent  = -1;141    // temporary store of child tasks for scheduling142    // note: accessing to elements is invalid after the task is moved to server_slot143    std::vector<server_task> child_tasks;144 145    // used by SERVER_TASK_TYPE_INFERENCE146    task_params   params;147    server_tokens tokens;148 149    // only used by CLI, this allow tokenizing CLI inputs on server side150    // we need this because mtmd_context and vocab are not accessible outside of server_context151    bool                    cli = false;152    std::string             cli_prompt;153    std::vector<raw_buffer> cli_files;154 155    server_task_type type;156 157    // used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE158    struct slot_action {159        int id_slot;160        std::string filename;161        std::string filepath;162    };163    slot_action slot_action;164 165    // used by SERVER_TASK_TYPE_METRICS166    bool metrics_reset_bucket = false;167 168    // used by SERVER_TASK_TYPE_SET_LORA169    std::map<int, float> set_lora; // mapping adapter ID -> scale170 171    server_task() = default;172 173    server_task(server_task_type type) : type(type) {}174 175    int32_t n_tokens() const {176        return tokens.size();177    }178 179    bool need_embd() const {180        switch (type) {181            case SERVER_TASK_TYPE_EMBEDDING:182            case SERVER_TASK_TYPE_RERANK:183                return true;184            default:185                return false;186        }187    }188 189    bool need_logits() const {190        switch (type) {191            case SERVER_TASK_TYPE_COMPLETION:192            case SERVER_TASK_TYPE_INFILL:193                return true;194            default:195                return false;196        }197    }198 199    bool need_sampling() const {200        switch (type) {201            case SERVER_TASK_TYPE_COMPLETION:202            case SERVER_TASK_TYPE_INFILL:203                return true;204            default:205                return false;206        }207    }208 209    static task_params params_from_json_cmpl(210        const llama_vocab * vocab,211        const common_params & params_base,212        const int n_ctx_slot,213        const std::vector<llama_logit_bias> & logit_bias_eog,214        const json & data);215 216    // utility function217    static std::unordered_set<int> get_list_id(const std::vector<server_task> & tasks) {218        std::unordered_set<int> ids(tasks.size());219        for (size_t i = 0; i < tasks.size(); i++) {220            ids.insert(tasks[i].id);221            for (auto & child : tasks[i].child_tasks) {222                ids.insert(child.id);223            }224        }225        return ids;226    }227 228    void add_child(int id_parent, int id_child) {229        server_task copy;230 231        copy.id        = id_child;232        copy.id_parent = id_parent;233        copy.params    = params;234        copy.type      = type;235        copy.tokens    = tokens.clone();236        copy.id_slot   = -1; // child tasks cannot specify slot237 238        // use different sampling seed for each child239        // note: https://github.com/ggml-org/llama.cpp/pull/18700#discussion_r2675115723240        if (copy.params.sampling.seed != LLAMA_DEFAULT_SEED) {241            copy.params.sampling.seed += (uint32_t)child_tasks.size() + 1;242        }243 244        child_tasks.push_back(std::move(copy));245    }246 247    // the task will be moved into queue, then onto slots248    // however, the state must be kept by caller (e.g., HTTP thread)249    task_result_state create_state() const {250        return task_result_state(params.chat_parser_params);251    }252 253    bool is_parent() const {254        return child_tasks.size() > 0;255    }256 257    bool is_child() const {258        return id_parent != -1;259    }260};261 262struct result_timings {263    int32_t cache_n = -1;264 265    int32_t prompt_n = -1;266    double prompt_ms = 0.0;267    double prompt_per_token_ms = 0.0;268    double prompt_per_second = 0.0;269 270    int32_t predicted_n = -1;271    double predicted_ms = 0.0;272    double predicted_per_token_ms = 0.0;273    double predicted_per_second = 0.0;274 275    // Optional speculative metrics - only included when > 0276    int32_t draft_n = 0;277    int32_t draft_n_accepted = 0;278 279    json to_json() const;280};281 282struct result_prompt_progress {283    int32_t total = 0;284    int32_t cache = 0;285    int32_t processed = 0;286    int64_t time_ms = 0;287 288    json to_json() const;289};290 291struct server_task_result {292    int id           = -1;293    int id_slot      = -1;294 295    // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader296    size_t index = 0; // to be used for batched tasks297 298    virtual bool is_error() {299        // only used by server_task_result_error300        return false;301    }302    virtual bool is_stop() {303        // only used by server_task_result_cmpl_*304        return true;305    }306    virtual void update(task_result_state &) {307        // only used by server_task_result_cmpl_*308    }309    virtual json to_json() = 0;310    virtual ~server_task_result() = default;311};312 313// using shared_ptr for polymorphism of server_task_result314using server_task_result_ptr = std::unique_ptr<server_task_result>;315 316struct completion_token_output {317    llama_token tok;318    float prob;319    std::string text_to_send;320    struct prob_info {321        llama_token tok;322        std::string txt;323        float prob;324    };325    std::vector<prob_info> probs;326 327    json to_json(bool post_sampling_probs) const;328 329    static json probs_vector_to_json(const std::vector<completion_token_output> & probs, bool post_sampling_probs);330 331    static float logarithm(float x);332 333    static std::vector<unsigned char> str_to_bytes(const std::string & str);334 335};336 337struct server_task_result_cmpl_final : server_task_result {338    std::string content;339    llama_tokens tokens;340 341    bool stream;342    bool include_usage;343    result_timings timings;344    std::string prompt;345 346    bool truncated;347    int32_t n_decoded;348    int32_t n_prompt_tokens;349    int32_t n_prompt_tokens_cache;350    int32_t n_tokens_cached;351    bool has_new_line;352    std::string stopping_word;353    stop_type stop = STOP_TYPE_NONE;354 355    bool post_sampling_probs;356    std::vector<completion_token_output> probs_output;357    std::vector<std::string>  response_fields;358 359    task_params generation_params;360 361    // response formatting362    bool               verbose  = false;363    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;364    std::string        oaicompat_model;365    std::string        oaicompat_cmpl_id;366    common_chat_msg    oaicompat_msg; // to be populated by update()367 368    std::vector<common_chat_msg_diff> oaicompat_msg_diffs; // to be populated by update()369    bool is_updated = false;370 371    // for OpenAI Responses API372    std::string oai_resp_id;373    std::string oai_resp_reasoning_id;374    std::string oai_resp_message_id;375 376    virtual bool is_stop() override {377        return true; // in stream mode, final responses are considered stop378    }379 380    virtual json to_json() override;381 382    virtual void update(task_result_state & state) override {383        is_updated = true;384        oaicompat_msg = state.update_chat_msg(content, false, oaicompat_msg_diffs);385 386        oai_resp_id = state.oai_resp_id;387        oai_resp_reasoning_id = state.oai_resp_reasoning_id;388        oai_resp_message_id = state.oai_resp_message_id;389    }390 391    json to_json_non_oaicompat();392 393    json usage_json_oaicompat();394 395    json to_json_oaicompat();396 397    json to_json_oaicompat_chat();398 399    json to_json_oaicompat_chat_stream();400 401    json to_json_oaicompat_resp();402 403    json to_json_oaicompat_resp_stream();404 405    json to_json_oaicompat_asr();406 407    json to_json_anthropic();408 409    json to_json_anthropic_stream();410};411 412struct server_task_result_cmpl_partial : server_task_result {413    std::string  content;414    llama_tokens tokens;415 416    int32_t n_decoded;417    int32_t n_prompt_tokens;418    int32_t n_prompt_tokens_cache;419 420    bool post_sampling_probs;421    bool is_progress = false;422    completion_token_output prob_output;423    result_timings timings;424    result_prompt_progress progress;425 426    // response formatting427    bool               verbose  = false;428    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;429    std::string        oaicompat_model;430    std::string        oaicompat_cmpl_id;431    std::vector<common_chat_msg_diff> oaicompat_msg_diffs; // to be populated by update()432    bool is_updated = false;433 434    // Streaming state copied from task_result_state for this chunk435    bool thinking_block_started = false;436    bool text_block_started     = false;437 438    // for OpenAI Responses API439    std::string oai_resp_id;440    std::string oai_resp_reasoning_id;441    std::string oai_resp_message_id;442    std::string oai_resp_fc_id;443 444    // for Anthropic API: track if any reasoning content has been generated445    bool anthropic_has_reasoning = false;446 447    virtual bool is_stop() override {448        return false; // in stream mode, partial responses are not considered stop449    }450 451    virtual void update(task_result_state & state) override;452 453    virtual json to_json() override;454 455    json to_json_non_oaicompat();456 457    json to_json_oaicompat();458 459    json to_json_oaicompat_chat();460 461    json to_json_oaicompat_resp();462 463    json to_json_oaicompat_asr();464 465    json to_json_anthropic();466};467 468struct server_task_result_embd : server_task_result {469    std::vector<std::vector<float>> embedding;470 471    int32_t n_tokens;472 473    // response formatting474    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;475 476    virtual json to_json() override;477 478    json to_json_non_oaicompat();479 480    json to_json_oaicompat();481};482 483struct server_task_result_rerank : server_task_result {484    float score = -1e6;485 486    int32_t n_tokens;487 488    virtual json to_json() override;489};490 491struct server_task_result_error : server_task_result {492    error_type err_type = ERROR_TYPE_SERVER;493    std::string err_msg;494 495    // for ERROR_TYPE_EXCEED_CONTEXT_SIZE496    int32_t n_prompt_tokens = 0;497    int32_t n_ctx           = 0;498 499    virtual bool is_error() override {500        return true;501    }502 503    virtual json to_json() override;504};505 506struct server_task_result_metrics : server_task_result {507    int n_idle_slots;508    int n_processing_slots;509    int n_tasks_deferred;510    int64_t t_start;511 512    // TODO: somehow reuse server_metrics in the future, instead of duplicating the fields513    uint64_t n_prompt_tokens_processed_total = 0;514    uint64_t t_prompt_processing_total       = 0;515    uint64_t n_tokens_predicted_total        = 0;516    uint64_t t_tokens_generation_total       = 0;517 518    uint64_t n_tokens_max = 0;519 520    uint64_t n_prompt_tokens_processed = 0;521    uint64_t t_prompt_processing       = 0;522 523    uint64_t n_tokens_predicted  = 0;524    uint64_t t_tokens_generation = 0;525 526    uint64_t n_decode_total     = 0;527    uint64_t n_busy_slots_total = 0;528 529    // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy530    // therefore, we use json to temporarily store the slot.to_json() result531    json slots_data = json::array();532 533    virtual json to_json() override;534};535 536struct server_task_result_slot_save_load : server_task_result {537    std::string filename;538    bool is_save; // true = save, false = load539 540    size_t n_tokens;541    size_t n_bytes;542    double t_ms;543 544    virtual json to_json() override;545};546 547struct server_task_result_slot_erase : server_task_result {548    size_t n_erased;549 550    virtual json to_json() override;551};552 553struct server_task_result_get_lora : server_task_result {554    struct lora {555        common_adapter_lora_info info;556        std::string  alora_invocation_string;557        llama_tokens alora_invocation_tokens;558    };559    std::vector<lora> loras;560 561    virtual json to_json() override;562};563 564struct server_task_result_apply_lora : server_task_result {565    virtual json to_json() override;566};567 568struct server_prompt_checkpoint {569    llama_pos pos_min;570    llama_pos pos_max;571 572    int64_t n_tokens;573 574    std::vector<uint8_t> data;575 576    size_t size() const {577        return data.size();578    }579 580    bool empty() const {581        return data.empty();582    }583 584    void clear() {585        pos_min = 0;586        pos_max = 0;587        n_tokens = 0;588        data.clear();589    }590};591 592struct server_prompt {593    server_tokens tokens;594 595    std::vector<uint8_t> data;596 597    std::list<server_prompt_checkpoint> checkpoints;598 599    size_t size() const {600        size_t res = data.size();601 602        for (const auto & checkpoint : checkpoints) {603            res += checkpoint.size();604        }605 606        return res;607    }608 609    int n_tokens() const {610        return tokens.size();611    }612 613    server_prompt clone() const {614        return server_prompt {615            tokens.clone(),616            data,617            checkpoints618        };619    }620};621 622struct server_prompt_cache {623    server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {624        this->limit_size   = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);625        this->limit_tokens = limit_tokens;626    }627 628    std::list<server_prompt> states;629 630    // in bytes, 0 = no limit631    size_t limit_size = 0;632 633    // in tokens, 0 = no limit634    size_t limit_tokens = 0;635 636    size_t size() const;637 638    size_t n_tokens() const;639 640    server_prompt * alloc(const server_prompt & prompt, size_t state_size);641 642    bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx, int32_t id_slot);643 644    void update();645};646