Felipe97/llama-cpp-compiled

1#pragma once2 3#include "common.h"4#include "llama.h"5 6#include <string>7#include <unordered_set>8#include <list>9#include <map>10 11// TODO: prevent including the whole server-common.h as we only use server_tokens12#include "server-common.h"13 14 15enum server_task_type {16    SERVER_TASK_TYPE_COMPLETION,17    SERVER_TASK_TYPE_EMBEDDING,18    SERVER_TASK_TYPE_RERANK,19    SERVER_TASK_TYPE_INFILL,20    SERVER_TASK_TYPE_CANCEL,21    SERVER_TASK_TYPE_CONTROL,22    SERVER_TASK_TYPE_NEXT_RESPONSE,23    SERVER_TASK_TYPE_METRICS,24    SERVER_TASK_TYPE_SLOT_GET,25    SERVER_TASK_TYPE_SLOT_SAVE,26    SERVER_TASK_TYPE_SLOT_RESTORE,27    SERVER_TASK_TYPE_SLOT_ERASE,28    SERVER_TASK_TYPE_GET_LORA,29    SERVER_TASK_TYPE_SET_LORA,30};31 32// TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common33enum task_response_type {34    TASK_RESPONSE_TYPE_NONE, // llama.cpp native format35    TASK_RESPONSE_TYPE_OAI_CHAT,36    TASK_RESPONSE_TYPE_OAI_CMPL,37    TASK_RESPONSE_TYPE_OAI_RESP,38    TASK_RESPONSE_TYPE_OAI_ASR, // transcriptions API39    TASK_RESPONSE_TYPE_OAI_EMBD,40    TASK_RESPONSE_TYPE_ANTHROPIC,41};42 43enum stop_type {44    STOP_TYPE_NONE,45    STOP_TYPE_EOS,46    STOP_TYPE_WORD,47    STOP_TYPE_LIMIT,48};49 50struct task_params {51    bool stream          = false;52    bool include_usage   = false;53    bool cache_prompt    = true; // remember the prompt to avoid reprocessing all prompt54    bool return_tokens   = false;55    bool return_progress = false;56 57    int32_t sse_ping_interval = 30; // seconds between SSE comment pings while the stream stays silent, -1 disables58 59    int32_t n_keep    =  0; // number of tokens to keep from initial prompt60    int32_t n_discard =  0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half61    int32_t n_predict = -1; // new tokens to predict62    int32_t n_indent  =  0; // minimum line indentation for the generated text in number of whitespace characters63    int32_t n_cmpl    =  1; // number of completions to generate from this prompt64 65    int32_t n_cache_reuse = 0; // min chunk size to attempt reusing from the cache via KV shifting (0 = disabled)66 67    int64_t t_max_prompt_ms  = -1; // TODO: implement68    int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit69 70    std::map<int, float> lora; // mapping adapter ID -> scale71 72    std::vector<std::string> antiprompt;73    std::vector<std::string> response_fields;74 75    bool timings_per_token   = false;76    bool post_sampling_probs = false;77 78    struct common_params_sampling sampling;79    struct common_params_speculative speculative;80 81    // response formatting82    bool               verbose  = false;83    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;84    std::string        oaicompat_model;85    std::string        oaicompat_cmpl_id;86 87    // realtime control (SERVER_TASK_TYPE_CONTROL)88    std::string        control_action;89    std::string        control_cmpl_id;90 91    // per-request parameters for chat parsing92    common_chat_parser_params chat_parser_params;93 94    // message spans for checkpointing95    common_chat_msg_spans message_spans;96 97    // Embeddings98    int32_t embd_normalize = 2; // (-1=none, 0=max absolute int16, 1=taxicab, 2=Euclidean/L2, >2=p-norm)99 100    json format_logit_bias(const std::vector<llama_logit_bias> & logit_bias) const;101    json to_json(bool only_metrics = false) const;102};103 104// struct for tracking the state of a task (e.g., for streaming)105struct task_result_state {106    // tracking diffs for partial tool calls107    std::vector<common_chat_msg_diff> diffs;108    common_chat_parser_params chat_parser_params;109    common_chat_msg chat_msg;110    std::string generated_text; // append new chunks of generated text here111    std::vector<std::string> generated_tool_call_ids;112    std::unordered_set<size_t> sent_tool_call_names;113 114    // for OpenAI Responses and Anthropic streaming API:115    // track output item / content block state across chunks116    bool thinking_block_started = false;117    bool text_block_started = false;118 119    // for OpenAI Responses streaming API120    bool oai_resp_created = false;121    const std::string oai_resp_id;122    const std::string oai_resp_reasoning_id;123    const std::string oai_resp_message_id;124    std::string oai_resp_fc_id; // function call ID for current args delta125 126    task_result_state(const common_chat_parser_params & chat_parser_params);127 128    // parse partial tool calls and update the internal state129    common_chat_msg update_chat_msg(130        const std::string & text_added,131        bool is_partial,132        std::vector<common_chat_msg_diff> & diffs,133        bool filter_tool_calls = false);134};135 136struct server_task {137    int id = -1; // to be filled by server_queue138 139    // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader140    size_t index = 0; // used when there are multiple prompts (batch request)141 142    // used by SERVER_TASK_TYPE_CANCEL143    int id_target = -1;144    int id_slot   = -1;145 146    // used by parallel sampling (multiple completions from same prompt)147    int id_parent  = -1;148    // temporary store of child tasks for scheduling149    // note: accessing to elements is invalid after the task is moved to server_slot150    std::vector<server_task> child_tasks;151 152    // used by SERVER_TASK_TYPE_INFERENCE153    task_params   params;154    server_tokens tokens;155 156    // only used by CLI, this allow tokenizing CLI inputs on server side157    // we need this because mtmd_context and vocab are not accessible outside of server_context158    bool                    cli = false;159    std::string             cli_prompt;160    std::vector<raw_buffer> cli_files;161 162    server_task_type type;163 164    // used by SERVER_TASK_TYPE_SLOT_SAVE, SERVER_TASK_TYPE_SLOT_RESTORE, SERVER_TASK_TYPE_SLOT_ERASE165    struct slot_action {166        int id_slot;167        std::string filename;168        std::string filepath;169    };170    slot_action slot_action;171 172    // used by SERVER_TASK_TYPE_METRICS173    bool metrics_reset_bucket = false;174 175    // used by SERVER_TASK_TYPE_SET_LORA176    std::map<int, float> set_lora; // mapping adapter ID -> scale177 178    server_task() = default;179 180    server_task(server_task_type type) : type(type) {}181 182    int32_t n_tokens() const {183        return tokens.size();184    }185 186    bool need_embd() const {187        switch (type) {188            case SERVER_TASK_TYPE_EMBEDDING:189            case SERVER_TASK_TYPE_RERANK:190                return true;191            default:192                return false;193        }194    }195 196    bool need_logits() const {197        switch (type) {198            case SERVER_TASK_TYPE_COMPLETION:199            case SERVER_TASK_TYPE_INFILL:200                return true;201            default:202                return false;203        }204    }205 206    bool need_sampling() const {207        switch (type) {208            case SERVER_TASK_TYPE_COMPLETION:209            case SERVER_TASK_TYPE_INFILL:210                return true;211            default:212                return false;213        }214    }215 216    // utility function217    static std::unordered_set<int> get_list_id(const std::vector<server_task> & tasks) {218        std::unordered_set<int> ids(tasks.size());219        for (size_t i = 0; i < tasks.size(); i++) {220            ids.insert(tasks[i].id);221            for (auto & child : tasks[i].child_tasks) {222                ids.insert(child.id);223            }224        }225        return ids;226    }227 228    void add_child(int id_parent, int id_child) {229        server_task copy;230 231        copy.id        = id_child;232        copy.id_parent = id_parent;233        copy.params    = params;234        copy.type      = type;235        copy.tokens    = tokens.clone();236        copy.id_slot   = -1; // child tasks cannot specify slot237 238        // use different sampling seed for each child239        // note: https://github.com/ggml-org/llama.cpp/pull/18700#discussion_r2675115723240        if (copy.params.sampling.seed != LLAMA_DEFAULT_SEED) {241            copy.params.sampling.seed += (uint32_t)child_tasks.size() + 1;242        }243 244        child_tasks.push_back(std::move(copy));245    }246 247    // the task will be moved into queue, then onto slots248    // however, the state must be kept by caller (e.g., HTTP thread)249    task_result_state create_state() const {250        return task_result_state(params.chat_parser_params);251    }252 253    bool is_parent() const {254        return child_tasks.size() > 0;255    }256 257    bool is_child() const {258        return id_parent != -1;259    }260};261 262struct result_prompt_progress {263    int32_t total = 0;264    int32_t cache = 0;265    int32_t processed = 0;266    int64_t time_ms = 0;267 268    json to_json() const;269};270 271struct server_task_result {272    int id           = -1;273    int id_slot      = -1;274 275    // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader276    size_t index = 0; // to be used for batched tasks277 278    virtual bool is_error() {279        // only used by server_task_result_error280        return false;281    }282    virtual bool is_stop() {283        // only used by server_task_result_cmpl_*284        return true;285    }286    virtual void update(task_result_state &) {287        // only used by server_task_result_cmpl_*288    }289    virtual json to_json() = 0;290    virtual ~server_task_result() = default;291    virtual server_task_result * clone() const {292        GGML_ABORT("not implemented for this task type");293    }294};295 296// using shared_ptr for polymorphism of server_task_result297using server_task_result_ptr = std::unique_ptr<server_task_result>;298 299struct completion_token_output {300    llama_token tok;301    float prob;302    std::string text_to_send;303    struct prob_info {304        llama_token tok;305        std::string txt;306        float prob;307    };308    std::vector<prob_info> probs;309 310    json to_json(bool post_sampling_probs) const;311 312    static json probs_vector_to_json(const std::vector<completion_token_output> & probs, bool post_sampling_probs);313 314    static float logarithm(float x);315 316    static std::vector<unsigned char> str_to_bytes(const std::string & str);317 318};319 320struct server_task_result_cmpl_final : server_task_result {321    std::string content;322    llama_tokens tokens;323 324    bool stream;325    bool include_usage;326    server_slot_stats stats;327    std::string prompt;328 329    bool truncated;330    int32_t n_decoded;331    int32_t n_prompt_tokens;332    int32_t n_prompt_tokens_cache;333    int32_t n_tokens_cached;334    bool has_new_line;335    std::string stopping_word;336    stop_type stop = STOP_TYPE_NONE;337 338    bool post_sampling_probs;339    std::vector<completion_token_output> probs_output;340    std::vector<std::string>  response_fields;341 342    task_params generation_params;343 344    // response formatting345    bool               verbose  = false;346    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;347    std::string        oaicompat_model;348    std::string        oaicompat_cmpl_id;349    common_chat_msg    oaicompat_msg; // to be populated by update()350 351    std::vector<common_chat_msg_diff> oaicompat_msg_diffs; // to be populated by update()352    bool is_updated = false;353 354    // for OpenAI Responses API355    std::string oai_resp_id;356    std::string oai_resp_reasoning_id;357    std::string oai_resp_message_id;358 359    virtual bool is_stop() override {360        return true; // in stream mode, final responses are considered stop361    }362 363    virtual json to_json() override;364 365    virtual void update(task_result_state & state) override {366        is_updated = true;367        oaicompat_msg = state.update_chat_msg(content, false, oaicompat_msg_diffs);368 369        oai_resp_id = state.oai_resp_id;370        oai_resp_reasoning_id = state.oai_resp_reasoning_id;371        oai_resp_message_id = state.oai_resp_message_id;372    }373 374    json to_json_non_oaicompat();375 376    json usage_json_oaicompat();377 378    json to_json_oaicompat();379 380    json to_json_oaicompat_chat();381 382    json to_json_oaicompat_chat_stream();383 384    json to_json_oaicompat_resp();385 386    json to_json_oaicompat_resp_stream();387 388    json to_json_oaicompat_asr();389 390    json to_json_anthropic();391 392    json to_json_anthropic_stream();393};394 395struct server_task_result_cmpl_partial : server_task_result {396    std::string  content;397    llama_tokens tokens;398 399    int32_t n_decoded;400    int32_t n_prompt_tokens;401    int32_t n_prompt_tokens_cache;402 403    bool post_sampling_probs;404    bool is_progress = false;405    bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)406                           // ref: https://github.com/ggml-org/llama.cpp/pull/23884407    completion_token_output prob_output;408    server_slot_stats stats;409    result_prompt_progress progress;410 411    // response formatting412    bool               verbose  = false;413    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;414    std::string        oaicompat_model;415    std::string        oaicompat_cmpl_id;416    std::vector<common_chat_msg_diff> oaicompat_msg_diffs; // to be populated by update()417    bool is_updated = false;418 419    // Streaming state copied from task_result_state for this chunk420    bool thinking_block_started = false;421    bool text_block_started     = false;422 423    // for OpenAI Responses API424    bool oai_resp_created = false;425    std::string oai_resp_id;426    std::string oai_resp_reasoning_id;427    std::string oai_resp_message_id;428    std::string oai_resp_fc_id;429 430    // for Anthropic API: track if any reasoning content has been generated431    bool anthropic_has_reasoning = false;432 433    virtual bool is_stop() override {434        return false; // in stream mode, partial responses are not considered stop435    }436 437    virtual void update(task_result_state & state) override;438 439    virtual json to_json() override;440 441    json to_json_non_oaicompat();442 443    json to_json_oaicompat();444 445    json to_json_oaicompat_chat();446 447    json to_json_oaicompat_resp();448 449    json to_json_oaicompat_asr();450 451    json to_json_anthropic();452};453 454struct server_task_result_embd : server_task_result {455    std::vector<std::vector<float>> embedding;456 457    int32_t n_tokens;458 459    // response formatting460    task_response_type res_type = TASK_RESPONSE_TYPE_NONE;461 462    virtual json to_json() override;463 464    json to_json_non_oaicompat();465 466    json to_json_oaicompat();467};468 469struct server_task_result_rerank : server_task_result {470    float score = -1e6;471 472    int32_t n_tokens;473 474    virtual json to_json() override;475};476 477struct server_task_result_error : server_task_result {478    error_type err_type = ERROR_TYPE_SERVER;479    std::string err_msg;480 481    // for ERROR_TYPE_EXCEED_CONTEXT_SIZE482    int32_t n_prompt_tokens = 0;483    int32_t n_ctx           = 0;484 485    virtual bool is_error() override {486        return true;487    }488 489    virtual json to_json() override;490};491 492// used by /metrics API493struct server_task_result_metrics : server_task_result {494    // these are immediate stats, not accumulated (server_metrics is cumulative)495    int n_processing_slots = 0;496    int n_tasks_deferred = 0;497 498    server_metrics metrics;499 500    virtual json to_json() override;501 502    struct metric_item {503        std::string name;504        std::string description;505        double value; // prometheus values are always float64506    };507    std::string to_metrics();508};509 510// used by /slots API511struct server_task_result_slots : server_task_result {512    int n_idle_slots = 0;513 514    // while we can also use std::vector<server_slot> this requires copying the slot object which can be quite messy515    // therefore, we use json to temporarily store the slot.to_json() result516    json slots_data = json::array();517 518    virtual json to_json() override;519};520 521struct server_task_result_slot_save_load : server_task_result {522    std::string filename;523    bool is_save; // true = save, false = load524 525    size_t n_tokens;526    size_t n_bytes;527    double t_ms;528 529    virtual json to_json() override;530};531 532struct server_task_result_slot_erase : server_task_result {533    size_t n_erased;534 535    virtual json to_json() override;536};537 538struct server_task_result_control : server_task_result {539    bool        success = false;540    std::string message; // optional detail when success is false541 542    virtual json to_json() override {543        json out = json { { "success", success } };544        if (!message.empty()) {545            out["message"] = message;546        }547        return out;548    }549};550 551struct server_task_result_get_lora : server_task_result {552    struct lora {553        common_adapter_lora_info info;554        std::string  alora_invocation_string;555        llama_tokens alora_invocation_tokens;556    };557    std::vector<lora> loras;558 559    virtual json to_json() override;560};561 562struct server_task_result_apply_lora : server_task_result {563    virtual json to_json() override;564};565 566struct server_prompt {567    server_tokens tokens;568 569    std::list<common_prompt_checkpoint> checkpoints;570 571    void clear() {572        tokens.clear();573        checkpoints.clear();574    }575 576    int n_tokens() const {577        return tokens.size();578    }579 580    server_prompt clone() const {581        return server_prompt {582            tokens.clone(),583            checkpoints,584        };585    }586};587 588struct server_prompt_data {589    std::vector<uint8_t> main;590    std::vector<uint8_t> drft;591 592    size_t size() const {593        return main.size() + drft.size();594    }595};596 597struct server_prompt_cache_state {598    server_prompt prompt;599    server_prompt_data data;600 601    size_t size() const {602        size_t res = data.size();603 604        for (const auto & ckpt : prompt.checkpoints) {605            res += ckpt.size();606        }607 608        return res;609    }610};611 612struct server_prompt_cache {613    server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {614        this->limit_size   = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);615        this->limit_tokens = limit_tokens;616    }617 618    std::list<server_prompt_cache_state> states;619 620    // in bytes, 0 = no limit621    size_t limit_size = 0;622 623    // in tokens, 0 = no limit624    size_t limit_tokens = 0;625 626    size_t size() const;627 628    size_t n_tokens() const;629 630    server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft);631 632    bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot);633 634    void update();635};636 637// used exclusively by router mode638struct server_task_result_router : server_task_result {639    json data;640    virtual json to_json() override { return data; }641    virtual server_task_result * clone() const override {642        return new server_task_result_router(*this);643    }644};645