Felipe97/llama-cpp-compiled
01.1k
1#pragma once2 3#include "common.h"4#include "log.h"5#include "llama.h"6#include "chat.h"7#include "mtmd.h"8#include "mtmd-helper.h"9#include "subproc.h"10 11#include "json.h"12 13#include <atomic>14#include <chrono>15#include <condition_variable>16#include <cinttypes>17#include <cstdio>18#include <functional>19#include <mutex>20#include <queue>21#include <string>22#include <vector>23 24using json = common_json;25 26#define SLT_DBG(slot, fmt, ...) LOG_DBG("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)27#define SLT_TRC(slot, fmt, ...) LOG_TRC("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)28#define SLT_INF(slot, fmt, ...) LOG_INF("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)29#define SLT_WRN(slot, fmt, ...) LOG_WRN("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)30#define SLT_ERR(slot, fmt, ...) LOG_ERR("slot %12.*s: id %2d | task %d | " fmt, 12, __func__, (slot).id, ((slot).task ? (slot).task->id : -1), __VA_ARGS__)31#define SLT_CNT(slot, fmt, ...) LOG_CNT("" fmt, __VA_ARGS__)32 33#define SRV_DBG(fmt, ...) LOG_DBG("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)34#define SRV_TRC(fmt, ...) LOG_TRC("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)35#define SRV_INF(fmt, ...) LOG_INF("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)36#define SRV_WRN(fmt, ...) LOG_WRN("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)37#define SRV_ERR(fmt, ...) LOG_ERR("srv %12.*s: " fmt, 12, __func__, __VA_ARGS__)38#define SRV_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__)39 40using raw_buffer = std::vector<uint8_t>;41 42template <typename T>43static T json_value(const json & body, const std::string & key, const T & default_value) {44 // Fallback null to default value45 if (body.contains(key) && !body.at(key).is_null()) {46 try {47 return body.at(key).get<T>();48 } catch (const common_json_error & err) {49 LOG_WRN("Wrong type supplied for parameter '%s', using default value: %s\n", key.c_str(), err.what());50 return default_value;51 }52 } else {53 return default_value;54 }55}56 57// https://community.openai.com/t/openai-chat-list-of-error-codes-and-types/357791/1158enum error_type {59 ERROR_TYPE_INVALID_REQUEST,60 ERROR_TYPE_AUTHENTICATION,61 ERROR_TYPE_SERVER,62 ERROR_TYPE_NOT_FOUND,63 ERROR_TYPE_PERMISSION,64 ERROR_TYPE_UNAVAILABLE, // custom error65 ERROR_TYPE_NOT_SUPPORTED, // custom error66 ERROR_TYPE_EXCEED_CONTEXT_SIZE, // custom error67};68 69// thin wrapper around common_grammar_trigger with (de)serialization functions70struct server_grammar_trigger {71 common_grammar_trigger value;72 73 server_grammar_trigger() = default;74 server_grammar_trigger(const common_grammar_trigger & value) : value(value) {}75 server_grammar_trigger(const json & in) {76 value.type = (common_grammar_trigger_type) in.at("type").get<int>();77 value.value = in.at("value").get<std::string>();78 if (value.type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {79 value.token = (llama_token) in.at("token").get<int>();80 }81 }82 83 json to_json() const {84 json out {85 {"type", (int) value.type},86 {"value", value.value},87 };88 if (value.type == COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN) {89 out["token"] = (int) value.token;90 }91 return out;92 }93};94 95json format_error_response(const std::string & message, const enum error_type type);96 97//98// random string / id99//100 101std::string random_string();102std::string gen_chatcmplid();103std::string gen_tool_call_id();104 105// get a random marker; note: each time the server restarts, the marker will be different106const char * get_media_marker();107 108//109// lora utils110//111 112// check whether the given lora set has only aloras activated (empty => false)113bool lora_all_alora(const std::vector<common_adapter_lora_info> & loras);114 115// if the two sets of loras are different, they require a cache clear unless the116// change is only from aloras to aloras.117bool lora_should_clear_cache(118 const std::vector<common_adapter_lora_info> & current,119 const std::vector<common_adapter_lora_info> & next);120 121std::map<int, float> parse_lora_request(const json & data);122 123bool are_lora_equal(124 const std::vector<common_adapter_lora_info> & l1,125 const std::vector<common_adapter_lora_info> & l2);126 127// get the ids of all enabled loras128std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras);129 130//131// server_tokens132//133 134/**135 * server_tokens is a helper to manage the input tokens and image for the server.136 * it is made this way to simplify the logic of KV cache management.137 */138struct server_tokens {139 bool has_mtmd = false;140 141private: // disallow accessing these members directly, risking out-of-sync142 143 // map a **start** index in tokens to the image chunk144 // note: the order need to be in-sync with tokens145 std::map<size_t, mtmd::input_chunk_ptr> map_idx_to_media;146 147 // list of tokens148 // if the token is LLAMA_TOKEN_NULL, it indicates that this position is occupied by media chunk149 // otherwise, it is a normal text token150 // note: a non-text chunk can occupy multiple tokens (aka memory cells) in the token list151 // note(2): for M-RoPE, an image can occupy different number of pos; do not assume 1-to-1 mapping tokens <-> pos152 llama_tokens tokens;153 154 // for ex. with input of 5 text tokens and 2 images (each image occupies 3 tokens and 2 pos):155 // [0] [1] [2] [3] [4] [img0] [img0] [img0] [img1] [img1] [img1]156 // idx 0 1 2 3 4 5 6 7 8 9 10157 // pos 0 1 2 3 4 5 5 5 7 7 7158 // map_idx_to_media will contain: {5, img0}, {8, img1}159 160public:161 server_tokens() = default;162 ~server_tokens() = default;163 164 // Prevent copying165 // TODO: server_tokens should be copyable - remove this:166 server_tokens(const server_tokens&) = delete;167 server_tokens& operator=(const server_tokens&) = delete;168 169 // Allow moving (usually implicitly generated if members are movable)170 server_tokens(server_tokens&&) = default;171 server_tokens& operator=(server_tokens&&) = default;172 173 // Allow accessing elements using [] operator174 llama_token operator[](size_t index) { return tokens[index]; }175 const llama_token& operator[](size_t index) const { return tokens[index]; }176 177 server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd);178 server_tokens(const llama_tokens & tokens, bool has_mtmd);179 180 // for debugging181 std::string str() const;182 183 // the next position after n_tokens. if n_tokens < 0, return the next position after all tokens.184 llama_pos pos_next(int64_t n_tokens = -1) const;185 186 // number of tokens with position < max_pos187 size_t size_up_to_pos(llama_pos max_pos) const;188 189 const mtmd::input_chunk_ptr & find_chunk(size_t idx) const;190 191 // find next media chunk after idx192 // returns a pair of pointer to the chunk (nullptr if not found) and its start index in tokens193 std::pair<const mtmd::input_chunk_ptr *, size_t> find_next_media_chunk(size_t idx) const;194 195 void push_back(llama_token tok);196 197 // will create a copy of the chunk if it contains non-text data198 void push_back(const mtmd_input_chunk * chunk);199 200 // same as push_back, but media chunks are stored as placeholders (no image/audio data)201 // only use this if the chunk will never be encoded again (e.g. it is already in the KV cache)202 void push_back_placeholder(const mtmd_input_chunk * chunk);203 204 // appends server tokens, updates the media map. copies media chunks.205 void push_back(server_tokens & tokens);206 207 // for compatibility with context shift and prompt truncation208 void insert(const llama_tokens & inp_tokens);209 210 // for compatibility with speculative decoding, ctx shift211 const llama_tokens & get_tokens() const;212 213 llama_tokens get_text_tokens() const;214 215 std::vector<char> serialize() const;216 static server_tokens deserialize(const llama_tokens & packed, bool has_mtmd);217 218 // for compatibility with speculative decoding219 void set_token(llama_pos pos, llama_token id);220 221 size_t size() const { return tokens.size(); }222 223 bool empty() const { return tokens.empty(); }224 225 void clear() {226 map_idx_to_media.clear();227 tokens.clear();228 }229 230 void keep_first(size_t n);231 232 std::string detokenize(const llama_context * ctx, bool special) const;233 234 size_t get_common_prefix(const server_tokens & b) const;235 236 // split the tokens into message spans, skipping over media chunks237 common_chat_msg_spans find_message_spans(const common_chat_msg_delimiters & delims) const;238 239 // check text token IDs and the mapping between media chunks and token ranges240 bool validate(const struct llama_context * ctx) const;241 242 server_tokens clone() const;243};244 245 246//247// tokenizer and input processing utils248//249 250bool json_is_array_of_numbers(const json & data);251 252// is array having BOTH numbers & strings?253bool json_is_array_of_mixed_numbers_strings(const json & data);254 255// does array have any individual integers/tokens?256bool json_is_array_and_contains_numbers(const json & data);257 258// get value by path(key1 / key2)259json json_get_nested_values(const std::vector<std::string> & paths, const json & js);260 261/**262 * this handles 2 cases:263 * - only string, example: "string"264 * - mixed string and tokens, example: [12, 34, "string", 56, 78]265 */266llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt, bool add_special, bool parse_special);267 268// return the last index of character that can form a valid string269// if the last character is potentially cut in half, return the index before the cut270// if validate_utf8(text) == text.size(), then the whole text is valid utf8271size_t validate_utf8(const std::string& text);272 273// process mtmd prompt, return the server_tokens containing both text tokens and media chunks274// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)275server_tokens process_mtmd_prompt(276 mtmd_context * mctx,277 const std::string & prompt,278 const std::vector<raw_buffer> & files,279 const mtmd_helper_init_opt & init_opt,280 bool is_placeholder = false);281 282/**283 * break the input "prompt" object into multiple prompt if needed, then tokenize them284 * this supports these cases:285 * - "prompt": "string"286 * - "prompt": [12, 34, 56]287 * - "prompt": [12, 34, "string", 56, 78]288 * - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }289 * and multiple prompts (multi-tasks):290 * - "prompt": ["string1", "string2"]291 * - "prompt": ["string1", [12, 34, 56]]292 * - "prompt": [[12, 34, 56], [78, 90, 12]]293 * - "prompt": [[12, 34, "string", 56, 78], [12, 34, 56], { "prompt_string": "string", "multimodal_data": [ "base64" ]}]294 */295std::vector<server_tokens> tokenize_input_prompts(296 const llama_vocab * vocab,297 mtmd_context * mctx,298 const json & json_prompt,299 bool add_special,300 bool parse_special,301 const mtmd_helper_init_opt & init_opt);302 303//304// OAI utils305//306 307// global server parameters for chat formatting / parsing308struct server_chat_params {309 bool use_jinja;310 bool prefill_assistant;311 common_reasoning_format reasoning_format;312 std::map<std::string, std::string> chat_template_kwargs; // mapping key --> json value313 common_chat_templates_ptr tmpls;314 bool allow_image;315 bool allow_audio;316 bool allow_video;317 bool enable_thinking = true;318 int reasoning_budget = -1;319 std::string reasoning_budget_message;320 std::string media_path;321 bool force_pure_content = false;322};323 324// used by /completions endpoint325json oaicompat_completion_params_parse(const json & body);326 327// used by /chat/completions endpoint328json oaicompat_chat_params_parse(329 json & body, /* openai api json semantics */330 const server_chat_params & opt,331 std::vector<raw_buffer> & out_files);332 333// TODO: move it to server-task.cpp334json format_embeddings_response_oaicompat(335 const json & request,336 const std::string & model_name,337 const json & embeddings,338 bool use_base64 = false);339 340// TODO: move it to server-task.cpp341json format_response_rerank(342 const json & request,343 const std::string & model_name,344 const json & ranks,345 bool is_tei_format,346 std::vector<std::string> & texts,347 int top_n);348 349//350// stats and metrics351//352 353// shared between server_slot and server_task_result_*354struct server_slot_stats {355 uint64_t n_prompt_cached = 0;356 uint64_t n_prompt_processed = 0;357 uint64_t n_gen = 0;358 359 // speculative decoding stats360 // note: the per-position breakdown lives in server_slot, it is not needed in a task result361 uint64_t n_draft_tokens = 0;362 uint64_t n_draft_accepted = 0;363 uint64_t n_draft_verif_steps = 0;364 365 // these are absolute timestamps (in us)366 // note: must be signed - they are subtracted before the later ones are set367 int64_t t_start = 0;368 int64_t t_prompt_last = 0;369 int64_t t_gen_last = 0;370 371 // can only move one direction: start -> prompt -> gen372 void update_prompt_start() {373 GGML_ASSERT(t_start == 0);374 t_start = ggml_time_us();375 }376 void set_prompt_last(int64_t t_us) {377 GGML_ASSERT(t_start > 0);378 t_prompt_last = t_us;379 }380 void update_prompt_last() {381 set_prompt_last(ggml_time_us());382 }383 void update_gen_last() {384 GGML_ASSERT(t_prompt_last > 0);385 t_gen_last = ggml_time_us();386 }387 388 // these are time durations389 int64_t t_elapsed_us() const {390 return ggml_time_us() - t_start;391 }392 double t_prompt_ms() const {393 if (t_prompt_last == 0) {394 return 0.0; // the prompt is not processed yet395 }396 return (t_prompt_last - t_start) / 1000.0;397 }398 int64_t t_gen_us() const {399 if (t_gen_last == 0) {400 return 0; // the generation is not started yet401 }402 // clamp to 1 us, the first token can land in the same us as t_prompt_last403 return std::max<int64_t>(1, t_gen_last - t_prompt_last);404 }405 double t_gen_ms() const {406 return t_gen_us() / 1000.0;407 }408 409 // number of decode steps spent on generation410 // the first token is free, it comes from the logits of the last prompt batch411 uint64_t n_gen_steps() const {412 return n_gen > 0 ? n_gen - 1 : 0;413 }414 415 // other derived metrics416 // note: all of them return 0.0 if the divisor is not known yet417 double t_prompt_per_token_ms() const {418 return n_prompt_processed > 0 ? t_prompt_ms() / n_prompt_processed : 0.0;419 }420 double t_gen_per_token_ms() const {421 return n_gen_steps() > 0 ? t_gen_ms() / n_gen_steps() : 0.0;422 }423 double n_prompt_tps() const {424 const double t_ms = t_prompt_ms();425 return t_ms > 0.0 ? 1e3 / t_ms * n_prompt_processed : 0.0;426 }427 double n_gen_tps() const {428 const double t_ms = t_gen_ms();429 return t_ms > 0.0 ? 1e3 / t_ms * n_gen_steps() : 0.0;430 }431 432 // false if the slot never started, i.e. the task result carries no stats433 bool is_set() const {434 return t_start > 0;435 }436 437 json to_json() const;438};439 440// shared between server_context_impl and server_task_result_*441// unlike server_slot_stats, server_metrics is server-global and cumulative, not tied to a slot442struct server_metrics {443 int64_t t_start = 0;444 445 struct bucket {446 uint64_t count = 0; // number of tokens447 uint64_t steps = 0; // number of decode steps,448 // this excludes first generated token (logits from prompt batch)449 uint64_t time = 0; // in microseconds450 451 // the rate uses the decode steps, so that "free" tokens do not inflate it452 double n_per_second() const {453 return time > 0 ? (double) steps / (double) time * 1e6 : 0.0;454 }455 456 void add(uint64_t n, uint64_t n_steps, uint64_t t_us) {457 count += n;458 steps += n_steps;459 time += t_us;460 }461 };462 463 // these are reset by reset_bucket(), only the rate is read from them464 bucket prompt_bucket;465 bucket predict_bucket;466 467 // metrics below are cumulative since the server started468 bucket prompt; // only processed tokens, cached ones are counted separately below469 bucket predict;470 471 // tokens reused from the cache need no decode, so they only have a count472 uint64_t n_prompt_cached = 0;473 474 uint64_t n_tokens_max = 0;475 476 uint64_t n_decode = 0;477 uint64_t n_busy_slots = 0;478 479 uint64_t n_draft_tokens = 0; // Total draft tokens generated480 uint64_t n_draft_accepted = 0; // Draft tokens actually accepted481 uint64_t n_draft_verif_steps = 0; // Total draft token verification steps by the target model482 std::vector<uint64_t> n_accepted_per_pos; // Accepted tokens per draft position483 484 void init() {485 t_start = ggml_time_us();486 }487 488 void reset_bucket() {489 prompt_bucket = {};490 predict_bucket = {};491 }492 493 void add_prompt(uint64_t n_tokens, uint64_t t_us) {494 prompt .add(n_tokens, n_tokens, t_us);495 prompt_bucket.add(n_tokens, n_tokens, t_us);496 }497 498 void add_prompt_cached(uint64_t n_tokens) {499 n_prompt_cached += n_tokens;500 }501};502 503//504// other utils505//506 507std::vector<llama_token_data> get_token_probabilities(llama_context * ctx, int idx, size_t n_top);508 509std::string safe_json_to_str(const json & data);510 511std::string tokens_to_str(llama_context * ctx, const llama_tokens & tokens);512std::string tokens_to_str(const llama_vocab * vocab, const llama_tokens & tokens);513 514// format incomplete utf-8 multibyte character for output515std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token);516 517// format server-sent event (SSE), return the formatted string to send518// note: if data is a json array, it will be sent as multiple events, one per item519std::string format_oai_sse(const json & data);520 521std::string format_oai_resp_sse(const json & data);522 523// format Anthropic-style SSE with event types524std::string format_anthropic_sse(const json & data);525 526bool is_valid_utf8(const std::string & str);527 528//529// formatting output responses530// TODO: move these to server-task.cpp531//532 533llama_tokens format_prompt_infill(534 const llama_vocab * vocab,535 const json & input_prefix,536 const json & input_suffix,537 const json & input_extra,538 const int n_batch,539 const int n_predict,540 const int n_ctx,541 const bool spm_infill,542 const llama_tokens & tokens_prompt);543 544// format rerank task: [BOS]query[EOS][SEP]doc[EOS].545server_tokens format_prompt_rerank(546 const struct llama_model * model,547 const struct llama_vocab * vocab,548 mtmd_context * mctx,549 const std::string & query,550 const std::string & doc,551 const mtmd_helper_init_opt & init_opt);552 553// simple implementation of a pipe554// used for streaming data between threads555template<typename T>556struct server_pipe {557 std::mutex mutex;558 std::condition_variable cv;559 std::queue<T> queue;560 std::atomic<bool> writer_closed{false};561 std::atomic<bool> reader_closed{false};562 563 // 0 = unbounded (default)564 // > 0, write() drops the oldest item once the queue is full565 size_t max_size = 0;566 567 void close_write() {568 writer_closed.store(true, std::memory_order_relaxed);569 cv.notify_all();570 }571 572 void close_read() {573 reader_closed.store(true, std::memory_order_relaxed);574 cv.notify_all();575 }576 577 // close_on_stop = true: should_stop means the reader is gone for good, so the writer is told the pipe is broken.578 // close_on_stop = false: should_stop is a per-read deadline and further reads still come, so the pipe stays usable.579 bool read(T & output, const std::function<bool()> & should_stop, bool close_on_stop = true) {580 std::unique_lock<std::mutex> lk(mutex);581 constexpr auto poll_interval = std::chrono::milliseconds(500);582 while (true) {583 if (!queue.empty()) {584 output = std::move(queue.front());585 queue.pop();586 return true;587 }588 if (writer_closed.load()) {589 return false; // clean EOF590 }591 if (should_stop && should_stop()) { // a null should_stop means "never stop"592 if (close_on_stop) {593 close_read(); // signal broken pipe to writer594 }595 return false; // cancelled / deadline reached596 }597 cv.wait_for(lk, poll_interval);598 }599 }600 601 bool write(T && data) {602 std::lock_guard<std::mutex> lk(mutex);603 if (reader_closed.load()) {604 return false; // broken pipe605 }606 if (max_size > 0) {607 while (queue.size() >= max_size) {608 queue.pop(); // drop oldest to stay bounded609 }610 }611 queue.push(std::move(data));612 cv.notify_one();613 return true;614 }615};616 617// wrapper around common_subproc to manage a child server process618// mainly used by router mode619struct server_subproc {620 common_subproc sproc;621 std::atomic<bool> stopped{false}; // set by the monitor once the process exited and was reaped622 623 bool is_alive() { return sproc.alive(); }624 void terminate() { sproc.terminate(); }625 int join() { return sproc.join(); }626 627 // true if the child's combined stdout/stderr pipe is available (call after create())628 bool has_output();629 630 // non-blocking read631 // returns the number of bytes read, 0 when nothing is available, -1 when the pipe is closed or broken632 int read_output(char * buf, size_t len);633 634 // wait until one of a set of children has output, wake() is called, or a timeout passes635 struct waiter {636 waiter();637 ~waiter();638 639 // thread-safe; on Windows this is a no-op, wait() returns within 50 ms anyway640 void wake();641 642 // timeout_ms < 0 waits until data or wake(); ready[i] is set for each child with data (or a broken pipe)643 void wait(const std::vector<server_subproc *> & procs, std::vector<bool> & ready, int64_t timeout_ms);644 645 private:646 intptr_t wake_fd[2] = { -1, -1 }; // POSIX self-pipe647 };648 649private:650 intptr_t out_handle = -1; // fd on POSIX, HANDLE on Windows; taken lazily from sproc651};652