echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0762
1 2#include "server-context.h"3#include "server-common.h"4#include "server-http.h"5#include "server-task.h"6#include "server-queue.h"7 8#include "build-info.h"9#include "common.h"10#include "llama.h"11#include "log.h"12#include "sampling.h"13#include "speculative.h"14#include "mtmd.h"15#include "mtmd-helper.h"16 17#include <algorithm>18#include <cstddef>19#include <cinttypes>20#include <exception>21#include <memory>22#include <filesystem>23#include <utility>24 25// fix problem with std::min and std::max26#if defined(_WIN32)27#define WIN32_LEAN_AND_MEAN28#ifndef NOMINMAX29# define NOMINMAX30#endif31#include <windows.h>32#endif33 34using json = nlohmann::ordered_json;35 36constexpr int HTTP_POLLING_SECONDS = 1;37 38static server_prompt_checkpoint server_get_checkpoint(llama_context * ctx, int id, int64_t n_tokens, llama_pos pos_min = -1, llama_pos pos_max = -1) {39 if (pos_min == -1) {40 pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx), id);41 }42 if (pos_max == -1) {43 pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx), id);44 }45 46 const size_t checkpoint_size = llama_state_seq_get_size_ext(ctx, id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);47 48 auto cur = server_prompt_checkpoint {49 /*.pos_min = */ pos_min,50 /*.pos_max = */ pos_max,51 /*.n_tokens = */ n_tokens,52 /*.data = */ std::vector<uint8_t>(checkpoint_size),53 };54 55 const size_t n = llama_state_seq_get_data_ext(ctx, cur.data.data(), checkpoint_size, id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);56 if (n != checkpoint_size) {57 GGML_ABORT("checkpoint size mismatch: expected %zu, got %zu\n", checkpoint_size, n);58 }59 60 return cur;61}62 63// state diagram: https://github.com/ggml-org/llama.cpp/pull/928364enum slot_state {65 SLOT_STATE_IDLE,66 SLOT_STATE_WAIT_OTHER, // after assigning a task, but waiting for parent slot to process prompt67 SLOT_STATE_STARTED, // after assigning a task and about to process prompt68 SLOT_STATE_PROCESSING_PROMPT,69 SLOT_STATE_DONE_PROMPT,70 SLOT_STATE_GENERATING,71};72 73enum server_state {74 SERVER_STATE_LOADING_MODEL, // Server is starting up, model not fully loaded yet75 SERVER_STATE_READY, // Server is ready and model is loaded76};77 78struct server_slot {79 int id;80 81 llama_context * ctx = nullptr;82 83 common_context_seq_rm_type ctx_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;84 85 // multimodal86 mtmd_context * mctx = nullptr;87 88 // speculative decoding89 llama_tokens spec_draft;90 std::vector<int32_t> spec_i_batch;91 server_prompt_checkpoint spec_ckpt;92 common_speculative_ptr spec;93 94 // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state95 // see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-371017583796 std::unique_ptr<const server_task> task;97 std::unique_ptr<const server_task> task_prev; // used for debugging98 99 // used to determine the slot that has been used the longest100 int64_t t_last_used = -1;101 102 // generation props103 int32_t n_ctx = 0; // context size per slot104 int32_t n_keep = 0;105 int32_t n_decoded = 0;106 int32_t n_remaining = -1;107 int32_t i_batch = -1;108 109 int32_t n_prompt_tokens_cache = 0;110 int32_t n_prompt_tokens_processed = 0;111 112 size_t last_nl_pos = 0;113 114 std::string generated_text;115 std::string debug_generated_text;116 llama_tokens generated_tokens;117 118 std::vector<completion_token_output> generated_token_probs;119 120 bool has_next_token = true;121 bool has_new_line = false;122 bool truncated = false;123 124 stop_type stop;125 126 std::string stopping_word;127 128 // state129 slot_state state = SLOT_STATE_IDLE;130 131 server_prompt prompt;132 133 void prompt_save(server_prompt_cache & prompt_cache) const {134 GGML_ASSERT(prompt.data.size() == 0);135 136 const size_t cur_size = llama_state_seq_get_size_ext(ctx, id, 0);137 138 SRV_WRN(" - saving prompt with length %d, total state size = %.3f MiB\n",139 (int) prompt.tokens.size(), cur_size / (1024.0 * 1024.0));140 141 auto * cur = prompt_cache.alloc(prompt, cur_size);142 if (cur == nullptr) {143 return;144 }145 146 llama_state_seq_get_data_ext(ctx, cur->data.data(), cur_size, id, 0);147 }148 149 bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens) {150 bool res = prompt_cache.load(prompt, tokens, ctx, id);151 if (!res) {152 SLT_WRN(*this, "%s", "failed to load prompt from cache\n");153 }154 155 return res;156 }157 158 void prompt_clear(bool allow_processing) {159 if (!allow_processing) {160 GGML_ASSERT(!is_processing());161 }162 163 SLT_INF(*this, "clearing prompt with %zu tokens\n", prompt.tokens.size());164 165 llama_memory_seq_rm(llama_get_memory(ctx), id, -1, -1);166 prompt.tokens.clear();167 }168 169 std::vector<common_adapter_lora_info> lora;170 int32_t alora_invocation_start = -1;171 172 // sampling173 json json_schema;174 175 common_sampler_ptr smpl;176 177 llama_token sampled; // in speculative mode, this is the last accepted token178 179 // stats180 size_t n_sent_text = 0; // number of sent text character181 182 int64_t t_start_process_prompt;183 int64_t t_start_generation;184 185 double t_prompt_processing = 0.0; // ms186 double t_token_generation = 0.0; // ms187 188 std::function<void(int /* id_slot */)> callback_on_release;189 190 // Speculative decoding stats191 int32_t n_draft_total = 0; // Total draft tokens generated192 int32_t n_draft_accepted = 0; // Draft tokens actually accepted193 194 void reset() {195 SLT_DBG(*this, "%s", "\n");196 197 n_prompt_tokens_cache = 0;198 199 last_nl_pos = 0;200 generated_text = "";201 has_new_line = false;202 truncated = false;203 stop = STOP_TYPE_NONE;204 stopping_word = "";205 n_sent_text = 0;206 207 if (can_speculate()) {208 spec_draft.clear();209 spec_i_batch.clear();210 spec_ckpt.clear();211 }212 generated_tokens.clear();213 generated_token_probs.clear();214 json_schema = json();215 216 // clear speculative decoding stats217 n_draft_total = 0;218 n_draft_accepted = 0;219 220 task_prev = std::move(task);221 task.reset();222 223 llama_set_sampler(ctx, id, nullptr);224 225 // clear alora start226 alora_invocation_start = -1;227 }228 229 void init_sampler() const {230 common_sampler_reset(smpl.get());231 232 if (!task->need_sampling()) {233 return;234 }235 236 const int64_t t_start = ggml_time_us();237 238 int n_text = 0;239 240 for (int i = 0; i < (int) prompt.tokens.size(); i++) {241 const llama_token id = prompt.tokens[i];242 243 if (id != LLAMA_TOKEN_NULL) {244 common_sampler_accept(smpl.get(), id, false);245 n_text++;246 }247 }248 249 SLT_INF(*this, "init sampler, took %0.2f ms, tokens: text = %d, total = %d\n",250 (ggml_time_us() - t_start) / 1000.0, n_text, (int) prompt.tokens.size());251 }252 253 // if the context does not have a memory module then all embeddings have to be computed within a single ubatch254 // also we cannot split if the pooling would require any past tokens255 bool can_split() const {256 GGML_ASSERT(task);257 258 return259 !task->need_embd() ||260 (llama_get_memory(ctx) && llama_pooling_type(ctx) == LLAMA_POOLING_TYPE_LAST);261 }262 263 bool can_batch_with(server_slot & other_slot) const {264 GGML_ASSERT(task);265 266 return task->type == other_slot.task->type && are_lora_equal(lora, other_slot.lora);267 }268 269 bool has_budget(const common_params & global_params) {270 GGML_ASSERT(task);271 272 if (task->params.n_predict == -1 && global_params.n_predict == -1) {273 return true; // limitless274 }275 276 n_remaining = -1;277 278 if (task->params.n_predict != -1) {279 n_remaining = task->params.n_predict - n_decoded;280 } else if (global_params.n_predict != -1) {281 n_remaining = global_params.n_predict - n_decoded;282 }283 284 return n_remaining > 0; // no budget285 }286 287 bool is_processing() const {288 return state != SLOT_STATE_IDLE;289 }290 291 bool can_speculate() const {292 return !!spec;293 }294 295 void add_token(const completion_token_output & token) {296 if (!is_processing()) {297 SLT_WRN(*this, "%s", "slot is not processing\n");298 return;299 }300 301 generated_token_probs.push_back(token);302 }303 304 int get_n_draft_max() const {305 GGML_ASSERT(task);306 307 if (!can_speculate()) {308 return 0;309 }310 311 // determine the max draft that fits the current slot state312 int n_draft_max = task->params.speculative.n_max;313 314 // note: slot.prompt is not yet expanded with the `id` token sampled above315 // also, need to leave space for 1 extra token to allow context shifts316 n_draft_max = std::min(n_draft_max, n_ctx - prompt.n_tokens() - 2);317 318 if (n_remaining > 0) {319 n_draft_max = std::min(n_draft_max, n_remaining - 1);320 }321 322 SLT_DBG(*this, "max possible draft: %d\n", n_draft_max);323 324 if (n_draft_max < task->params.speculative.n_min) {325 SLT_DBG(*this, "the max possible draft is too small: %d < %d - skipping speculative decoding\n", n_draft_max, task->params.speculative.n_min);326 n_draft_max = 0;327 }328 329 return n_draft_max;330 }331 332 void update_batch(llama_batch & batch) {333 const int n_draft_max = get_n_draft_max();334 if (n_draft_max > 0) {335 GGML_ASSERT(can_speculate());336 337 // generate draft tokens in speculative decoding mode338 // TODO: rework to have a single draft llama_context shared across all slots [TAG_SERVER_SPEC_REWORK]339 // perform the speculative drafting for all sequences at the same time in a single batch340 const llama_tokens & tokens = prompt.tokens.get_text_tokens();341 342 const auto & params_spec = task->params.speculative;343 344 if (!spec_draft.empty()) {345 // we have a previous (partial) draft to reuse346 if (ctx_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL) {347 GGML_ASSERT(!spec_ckpt.empty());348 }349 } else {350 GGML_ASSERT(spec_i_batch.empty());351 352 // generate a new draft353 spec_draft = common_speculative_draft(spec.get(), params_spec, tokens, sampled);354 355 if (spec_draft.size() > (size_t) n_draft_max) {356 SLT_WRN(*this, "draft size %d exceeds max %d, truncating\n", (int) spec_draft.size(), n_draft_max);357 spec_draft.resize(n_draft_max);358 }359 360 if (spec_draft.size() < (size_t) params_spec.n_min) {361 SLT_DBG(*this, "ignoring small draft: %d < %d\n", (int) spec_draft.size(), params_spec.n_min);362 spec_draft.clear();363 }364 365 if (!spec_draft.empty() && ctx_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL) {366 const auto n_tokens = prompt.tokens.size();367 368 spec_ckpt = server_get_checkpoint(ctx, this->id, n_tokens);369 370 SLT_DBG(*this, "created speculative checkpoint (pos_min = %d, pos_max = %d, n_tokens = %zu, size = %.3f MiB)\n",371 spec_ckpt.pos_min, spec_ckpt.pos_max, n_tokens, (float) spec_ckpt.data.size() / 1024 / 1024);372 }373 }374 375 GGML_ASSERT(spec_draft.size() <= (size_t) n_draft_max);376 }377 378 if (spec_draft.empty()) {379 // no speculative decoding380 i_batch = batch.n_tokens;381 382 common_batch_add(batch, sampled, prompt.tokens.pos_next(), { this->id }, true);383 384 SLT_DBG(*this, "slot decode token, id=%d, n_ctx = %d, n_tokens = %d, truncated = %d\n",385 sampled, n_ctx, prompt.n_tokens(), truncated);386 } else {387 SLT_DBG(*this, "generate_draft: id=%d, #tokens=%zu, #draft=%zu, pos_next=%d\n",388 sampled, prompt.tokens.size(), spec_draft.size(), prompt.tokens.pos_next());389 390 GGML_ASSERT(spec_i_batch.empty());391 392 spec_i_batch.push_back(batch.n_tokens);393 for (size_t i = 0; i < spec_draft.size(); i++) {394 spec_i_batch.push_back(batch.n_tokens + i + 1);395 }396 397 auto pos0 = prompt.tokens.pos_next();398 399 common_batch_add(batch, sampled, pos0++, { this->id }, true);400 for (auto token : spec_draft) {401 common_batch_add(batch, token, pos0++, { this->id }, true);402 }403 }404 405 prompt.tokens.push_back(sampled);406 prompt.tokens.insert(spec_draft);407 }408 409 void release() {410 if (is_processing()) {411 GGML_ASSERT(task);412 413 SLT_INF(*this, "stop processing: n_tokens = %d, truncated = %d\n", prompt.n_tokens(), truncated);414 415 t_last_used = ggml_time_us();416 t_token_generation = (ggml_time_us() - t_start_generation) / 1e3;417 418 state = SLOT_STATE_IDLE;419 420 // do not keep context of the child slots - the parent's context is enough421 if (task->is_child()) {422 prompt_clear(false);423 }424 425 reset();426 427 callback_on_release(id);428 }429 }430 431 result_timings get_timings() const {432 result_timings timings;433 timings.cache_n = n_prompt_tokens_cache;434 435 timings.prompt_n = n_prompt_tokens_processed;436 timings.prompt_ms = t_prompt_processing;437 timings.prompt_per_token_ms = t_prompt_processing / n_prompt_tokens_processed;438 timings.prompt_per_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;439 440 timings.predicted_n = n_decoded;441 timings.predicted_ms = t_token_generation;442 timings.predicted_per_token_ms = t_token_generation / n_decoded;443 timings.predicted_per_second = 1e3 / t_token_generation * n_decoded;444 445 // Add speculative metrics446 if (n_draft_total > 0) {447 timings.draft_n = n_draft_total;448 timings.draft_n_accepted = n_draft_accepted;449 }450 451 return timings;452 }453 454 size_t find_stopping_strings(const std::string & text, const size_t last_token_size, bool is_full_stop) {455 GGML_ASSERT(task);456 457 size_t stop_pos = std::string::npos;458 459 for (const std::string & word : task->params.antiprompt) {460 size_t pos;461 462 if (is_full_stop) {463 const size_t tmp = word.size() + last_token_size;464 const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;465 466 pos = text.find(word, from_pos);467 } else {468 // otherwise, partial stop469 pos = string_find_partial_stop(text, word);470 }471 472 if (pos != std::string::npos && (stop_pos == std::string::npos || pos < stop_pos)) {473 if (is_full_stop) {474 stop = STOP_TYPE_WORD;475 stopping_word = word;476 has_next_token = false;477 }478 stop_pos = pos;479 }480 }481 482 return stop_pos;483 }484 485 void print_timings() const {486 const double t_prompt = t_prompt_processing / n_prompt_tokens_processed;487 const double n_prompt_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;488 489 const double t_gen = t_token_generation / n_decoded;490 const double n_gen_second = 1e3 / t_token_generation * n_decoded;491 492 SLT_INF(*this,493 "\n"494 "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n"495 " eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n"496 " total time = %10.2f ms / %5d tokens\n",497 t_prompt_processing, n_prompt_tokens_processed, t_prompt, n_prompt_second,498 t_token_generation, n_decoded, t_gen, n_gen_second,499 t_prompt_processing + t_token_generation, n_prompt_tokens_processed + n_decoded);500 501 if (n_draft_total > 0) {502 const float draft_ratio = (float) n_draft_accepted / n_draft_total;503 SLT_CNT(*this,504 "draft acceptance rate = %0.5f (%5d accepted / %5d generated)\n",505 draft_ratio, n_draft_accepted, n_draft_total506 );507 }508 509 common_speculative_print_stats(spec.get());510 }511 512 json to_json(bool only_metrics = false) const {513 json res;514 515 res = {516 {"id", id},517 {"n_ctx", n_ctx},518 {"speculative", can_speculate()},519 {"is_processing", is_processing()},520 };521 522 const auto & ptask = task ? task : task_prev;523 524 if (ptask) {525 res["id_task"] = ptask->id;526 res["params"] = ptask->params.to_json(only_metrics);527 res["next_token"] = {528 {529 {"has_next_token", has_next_token},530 {"has_new_line", has_new_line},531 {"n_remain", n_remaining},532 {"n_decoded", n_decoded},533 }534 };535 536 if (!only_metrics) {537 res["prompt"] = ptask->tokens.detokenize(ctx, true);538 res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;539 }540 }541 542 return res;543 }544 545 void copy_state_to(server_slot & other) const {546 GGML_ASSERT(state == SLOT_STATE_DONE_PROMPT);547 548 llama_memory_seq_rm(llama_get_memory(ctx), other.id, -1, -1);549 llama_memory_seq_cp(llama_get_memory(ctx), id, other.id, -1, -1);550 551 other.n_decoded = n_decoded;552 other.n_remaining = n_remaining;553 other.i_batch = i_batch;554 555 other.t_start_process_prompt = t_start_process_prompt;556 other.t_prompt_processing = t_prompt_processing;557 other.n_prompt_tokens_cache = n_prompt_tokens_cache;558 other.n_prompt_tokens_processed = n_prompt_tokens_processed;559 560 other.prompt = prompt.clone();561 other.init_sampler();562 }563};564 565 566 567//568// server_metrics569//570 571struct server_metrics {572 int64_t t_start = 0;573 574 uint64_t n_prompt_tokens_processed_total = 0;575 uint64_t t_prompt_processing_total = 0;576 uint64_t n_tokens_predicted_total = 0;577 uint64_t t_tokens_generation_total = 0;578 579 uint64_t n_tokens_max = 0;580 581 uint64_t n_prompt_tokens_processed = 0;582 uint64_t t_prompt_processing = 0;583 584 uint64_t n_tokens_predicted = 0;585 uint64_t t_tokens_generation = 0;586 587 uint64_t n_decode_total = 0;588 uint64_t n_busy_slots_total = 0;589 590 void init() {591 t_start = ggml_time_us();592 }593 594 void on_prompt_eval(const server_slot & slot) {595 n_prompt_tokens_processed_total += slot.n_prompt_tokens_processed;596 n_prompt_tokens_processed += slot.n_prompt_tokens_processed;597 t_prompt_processing += slot.t_prompt_processing;598 t_prompt_processing_total += slot.t_prompt_processing;599 600 n_tokens_max = std::max(n_tokens_max, (uint64_t) slot.prompt.n_tokens());601 }602 603 void on_prediction(const server_slot & slot) {604 n_tokens_predicted_total += slot.n_decoded;605 n_tokens_predicted += slot.n_decoded;606 t_tokens_generation += slot.t_token_generation;607 t_tokens_generation_total += slot.t_token_generation;608 }609 610 void on_decoded(const std::vector<server_slot> & slots) {611 n_decode_total++;612 for (const auto & slot : slots) {613 if (slot.is_processing()) {614 n_busy_slots_total++;615 }616 n_tokens_max = std::max(n_tokens_max, (uint64_t) slot.prompt.n_tokens());617 }618 }619 620 void reset_bucket() {621 n_prompt_tokens_processed = 0;622 t_prompt_processing = 0;623 n_tokens_predicted = 0;624 t_tokens_generation = 0;625 }626};627 628 629//630// server_context_impl (private implementation)631//632 633struct server_context_impl {634 friend struct server_context;635 636public:637 // only use these pointers outside of this class:638 // - when not in sleeping state639 // - and, with thread-safe APIs (e.g., tokenizer calls)640 llama_model * model = nullptr;641 mtmd_context * mctx = nullptr;642 const llama_vocab * vocab = nullptr;643 644 server_queue queue_tasks;645 server_response queue_results;646 647 // note: chat_params must not be refreshed upon existing sleeping state648 server_chat_params chat_params;649 650 ~server_context_impl() {651 if (!sleeping) {652 // destroy() is already called when entering sleeping state653 // we don't call it again here to avoid double free654 destroy();655 }656 }657 658private:659 // note: accessing these fields outside of this class is not thread-safe660 // use server_context methods instead661 662 common_params params_base;663 664 // note: keep these alive - they determine the lifetime of the model, context, etc.665 common_init_result_ptr llama_init;666 667 llama_context * ctx = nullptr;668 669 llama_batch batch {};670 671 llama_model_ptr model_dft;672 673 bool add_bos_token = true;674 675 int32_t n_ctx; // total context for all clients / slots676 677 // slots / clients678 std::vector<server_slot> slots;679 680 int slots_debug = 0;681 int n_empty_consecutive = 0;682 683 std::unique_ptr<server_prompt_cache> prompt_cache;684 685 server_metrics metrics;686 687 json json_webui_settings = json::object();688 689 // Necessary similarity of prompt for slot selection690 float slot_prompt_similarity = 0.0f;691 692 std::string model_name; // name of the loaded model, to be used by API693 std::set<std::string> model_aliases; // additional names for the model694 std::set<std::string> model_tags; // informational tags695 696 bool sleeping = false;697 698 void destroy() {699 llama_init.reset();700 701 ctx = nullptr;702 model = nullptr;703 704 mtmd_free(mctx);705 mctx = nullptr;706 707 for (server_slot & slot : slots) {708 if (slot.can_speculate()) {709 slot.spec.reset();710 }711 }712 713 llama_batch_free(batch);714 }715 716 void slot_save_and_clear(server_slot & slot) {717 if (slot.prompt.n_tokens() == 0) {718 return;719 }720 SLT_INF(slot, "%s", "saving idle slot to prompt cache\n");721 SLT_DBG(slot, "%s", "__TEST_TAG_CLEAR_IDLE_SLOT__\n");722 slot.prompt_save(*prompt_cache);723 slot.prompt_clear(false);724 prompt_cache->update();725 }726 727 void handle_sleeping_state(bool new_state) {728 GGML_ASSERT(sleeping != new_state);729 if (new_state) {730 SRV_INF("%s", "server is entering sleeping state\n");731 destroy();732 } else {733 SRV_INF("%s", "server is exiting sleeping state\n");734 if (!load_model(params_base)) {735 GGML_ABORT("failed to reload model after sleeping");736 }737 }738 sleeping = new_state;739 }740 741 // load the model and initialize llama_context742 // this may also be called to resume from sleeping state743 bool load_model(common_params & params) {744 bool is_resume = sleeping;745 746 SRV_INF("loading model '%s'\n", params.model.path.c_str());747 748 params_base = params;749 750 llama_init = common_init_from_params(params_base);751 752 model = llama_init->model();753 ctx = llama_init->context();754 755 if (model == nullptr) {756 SRV_ERR("failed to load model, '%s'\n", params_base.model.path.c_str());757 return false;758 }759 760 vocab = llama_model_get_vocab(model);761 762 n_ctx = llama_n_ctx(ctx);763 764 add_bos_token = llama_vocab_get_add_bos(vocab);765 766 if (params_base.speculative.has_dft()) {767 // TODO speculative: move to common/speculative.cpp?768 SRV_INF("loading draft model '%s'\n", params_base.speculative.mparams_dft.path.c_str());769 770 const auto & params_spec = params_base.speculative;771 772 auto params_dft = params_base;773 774 params_dft.n_parallel = 1;775 params_dft.n_ctx = params_spec.n_ctx == 0 ? llama_n_ctx_seq(ctx) : params_spec.n_ctx;776 params_dft.n_batch = llama_n_ctx_seq(ctx);777 params_dft.devices = params_spec.devices;778 params_dft.model = params_spec.mparams_dft;779 params_dft.n_gpu_layers = params_spec.n_gpu_layers;780 params_dft.cache_type_k = params_spec.cache_type_k;781 params_dft.cache_type_v = params_spec.cache_type_v;782 783 if (params_spec.cpuparams.n_threads > 0) {784 params_dft.cpuparams.n_threads = params_spec.cpuparams.n_threads;785 params_dft.cpuparams_batch.n_threads = params_spec.cpuparams_batch.n_threads;786 }787 788 params_dft.tensor_buft_overrides = params_spec.tensor_buft_overrides;789 790 auto mparams_dft = common_model_params_to_llama(params_dft);791 792 model_dft.reset(llama_model_load_from_file(params_dft.model.path.c_str(), mparams_dft));793 if (model_dft == nullptr) {794 SRV_ERR("failed to load draft model, '%s'\n", params_dft.model.path.c_str());795 return false;796 }797 798 params_base.speculative.model_dft = model_dft.get();799 params_base.speculative.cparams_dft = common_context_params_to_llama(params_dft);800 }801 802 std::string & mmproj_path = params_base.mmproj.path;803 if (!mmproj_path.empty()) {804 if (!is_resume) {805 mtmd_helper_log_set(common_log_default_callback, nullptr);806 }807 808 mtmd_context_params mparams = mtmd_context_params_default();809 810 mparams.use_gpu = params_base.mmproj_use_gpu;811 mparams.print_timings = false;812 mparams.n_threads = params_base.cpuparams.n_threads;813 mparams.flash_attn_type = params_base.flash_attn_type;814 mparams.warmup = params_base.warmup;815 mparams.image_min_tokens = params_base.image_min_tokens;816 mparams.image_max_tokens = params_base.image_max_tokens;817 mparams.media_marker = get_media_marker();818 819 mctx = mtmd_init_from_file(mmproj_path.c_str(), model, mparams);820 if (mctx == nullptr) {821 SRV_ERR("failed to load multimodal model, '%s'\n", mmproj_path.c_str());822 return false;823 }824 SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str());825 826 if (params_base.ctx_shift) {827 params_base.ctx_shift = false;828 SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");829 }830 831 if (params_base.n_cache_reuse) {832 params_base.n_cache_reuse = 0;833 SRV_WRN("%s\n", "cache_reuse is not supported by multimodal, it will be disabled");834 }835 }836 837 if (!llama_memory_can_shift(llama_get_memory(ctx))) {838 if (params_base.ctx_shift) {839 params_base.ctx_shift = false;840 SRV_WRN("%s\n", "ctx_shift is not supported by this context, it will be disabled");841 }842 843 if (params_base.n_cache_reuse) {844 params_base.n_cache_reuse = 0;845 SRV_WRN("%s\n", "cache_reuse is not supported by this context, it will be disabled");846 }847 }848 849 if (llama_model_n_swa(model) == 0) {850 if (params_base.swa_full) {851 params_base.swa_full = false;852 SRV_WRN("%s\n", "swa_full is not supported by this model, it will be disabled");853 }854 }855 856 // Necessary similarity of prompt for slot selection857 slot_prompt_similarity = params_base.slot_prompt_similarity;858 859 // setup slots860 SRV_INF("initializing slots, n_slots = %d\n", params_base.n_parallel);861 862 const int n_ctx_train = llama_model_n_ctx_train(model);863 864 int n_ctx_slot = llama_n_ctx_seq(ctx);865 if (n_ctx_slot > n_ctx_train) {866 SRV_WRN("the slot context (%d) exceeds the training context of the model (%d) - capping\n", n_ctx_slot, n_ctx_train);867 n_ctx_slot = n_ctx_train;868 }869 870 slots.clear();871 872 const auto ctx_seq_rm_type = common_context_can_seq_rm(ctx);873 if (ctx_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_NO) {874 SRV_WRN("%s", "speculative decoding not supported by this context\n");875 }876 877 if (ctx_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL) {878 SRV_WRN("%s", "speculative decoding will use checkpoints\n");879 }880 881 // initialize slots882 for (int i = 0; i < params_base.n_parallel; i++) {883 slots.emplace_back();884 }885 886 for (int i = 0; i < params_base.n_parallel; i++) {887 server_slot & slot = slots[i];888 889 slot.id = i;890 slot.ctx = ctx;891 slot.n_ctx = n_ctx_slot;892 893 slot.ctx_seq_rm_type = ctx_seq_rm_type;894 895 slot.mctx = mctx;896 slot.prompt.tokens.has_mtmd = mctx != nullptr;897 898 // try speculative decoding899 if (ctx_seq_rm_type != COMMON_CONTEXT_SEQ_RM_TYPE_NO) {900 slot.spec.reset(common_speculative_init(params_base.speculative, slot.ctx));901 902 if (slot.spec) {903 SLT_INF(slot, "%s", "speculative decoding context initialized\n");904 }905 }906 907 SLT_INF(slot, "new slot, n_ctx = %d\n", slot.n_ctx);908 909 slot.callback_on_release = [this](int id_slot) {910 queue_tasks.pop_deferred_task(id_slot);911 };912 913 slot.reset();914 }915 916 {917 const char * LLAMA_SERVER_SLOTS_DEBUG = getenv("LLAMA_SERVER_SLOTS_DEBUG");918 slots_debug = LLAMA_SERVER_SLOTS_DEBUG ? atoi(LLAMA_SERVER_SLOTS_DEBUG) : 0;919 920 if (slots_debug) {921 SRV_WRN("slots debug = %d\n", slots_debug);922 }923 }924 925 // the update_slots() logic will always submit a maximum of n_batch or n_parallel tokens926 // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used)927 {928 const int32_t n_batch = llama_n_batch(ctx);929 batch = llama_batch_init(std::max(n_batch, params_base.n_parallel), 0, 1);930 }931 932 if (params_base.cache_ram_mib != 0) {933 if (params_base.cache_ram_mib < 0) {934 SRV_WRN("prompt cache is enabled, size limit: %s\n", "no limit");935 } else {936 SRV_WRN("prompt cache is enabled, size limit: %d MiB\n", params_base.cache_ram_mib);937 }938 SRV_WRN("%s", "use `--cache-ram 0` to disable the prompt cache\n");939 940 prompt_cache = std::make_unique<server_prompt_cache>(params_base.cache_ram_mib, n_ctx);941 } else {942 SRV_WRN("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");943 }944 SRV_WRN("%s", "for more info see https://github.com/ggml-org/llama.cpp/pull/16391\n");945 946 if (!params_base.model_alias.empty()) {947 // backward compat: use first alias as model name948 model_name = *params_base.model_alias.begin();949 } else if (!params_base.model.name.empty()) {950 model_name = params_base.model.name;951 } else {952 // fallback: derive model name from file name953 auto model_path = std::filesystem::path(params_base.model.path);954 model_name = model_path.filename().string();955 }956 957 model_aliases = params_base.model_alias;958 model_tags = params_base.model_tags;959 960 // propagate new defaults back to caller961 params = params_base;962 963 if (!is_resume) {964 return init();965 }966 967 return true;968 }969 970 // unlike load_model(), this is only called once during initialization971 bool init() {972 GGML_ASSERT(ctx != nullptr);973 GGML_ASSERT(model != nullptr);974 GGML_ASSERT(!sleeping);975 976 // wiring up server queues977 queue_tasks.on_new_task([this](server_task && task) {978 process_single_task(std::move(task));979 });980 queue_tasks.on_update_slots([this]() {981 update_slots();982 });983 queue_tasks.on_sleeping_state([this](bool sleeping) {984 handle_sleeping_state(sleeping);985 });986 987 metrics.init();988 989 if (params_base.cache_idle_slots) {990 if (!params_base.kv_unified) {991 SRV_WRN("%s: --cache-idle-slots requires --kv-unified, disabling\n", __func__);992 params_base.cache_idle_slots = false;993 } else if (params_base.cache_ram_mib == 0) {994 SRV_WRN("%s: --cache-idle-slots requires --cache-ram, disabling\n", __func__);995 params_base.cache_idle_slots = false;996 } else {997 SRV_INF("%s: idle slots will be saved to prompt cache and cleared upon starting a new task\n", __func__);998 SRV_DBG("%s", "__TEST_TAG_CLEAR_IDLE_ENABLED__\n");999 }1000 }1001 1002 // populate webui settings1003 {1004 if (!params_base.webui_config_json.empty()) {1005 try {1006 json_webui_settings = json::parse(params_base.webui_config_json);1007 } catch (const std::exception & e) {1008 SRV_ERR("%s: failed to parse webui config: %s\n", __func__, e.what());1009 return false;1010 }1011 }1012 }1013 1014 // populate chat template params1015 {1016 common_chat_templates_ptr chat_templates;1017 1018 try {1019 chat_templates = common_chat_templates_init(model, params_base.chat_template);1020 1021 LOG_INF("%s: chat template, example_format: '%s'\n", __func__,1022 common_chat_format_example(chat_templates.get(), params_base.use_jinja, params_base.default_template_kwargs).c_str());1023 1024 } catch (const std::exception & e) {1025 SRV_ERR("%s: chat template parsing error: %s\n", __func__, e.what());1026 SRV_ERR("%s: please consider disabling jinja via --no-jinja, or use a custom chat template via --chat-template\n", __func__);1027 SRV_ERR("%s: for example: --no-jinja --chat-template chatml\n", __func__);1028 return false;1029 }1030 1031 // thinking is enabled if:1032 // 1. It's not explicitly disabled via --reasoning off1033 // 2. The chat template supports it1034 const bool template_supports_thinking = params_base.use_jinja && common_chat_templates_support_enable_thinking(chat_templates.get());1035 const bool enable_thinking = params_base.enable_reasoning != 0 && template_supports_thinking;1036 SRV_INF("%s: chat template, thinking = %d\n", __func__, enable_thinking);1037 1038 chat_params = {1039 /* use_jinja */ params_base.use_jinja,1040 /* prefill_assistant */ params_base.prefill_assistant,1041 /* reasoning_format */ params_base.reasoning_format,1042 /* chat_template_kwargs */ params_base.default_template_kwargs,1043 /* tmpls */ std::move(chat_templates),1044 /* allow_image */ mctx ? mtmd_support_vision(mctx) : false,1045 /* allow_audio */ mctx ? mtmd_support_audio (mctx) : false,1046 /* enable_thinking */ enable_thinking,1047 /* reasoning_budget */ params_base.reasoning_budget,1048 /* reasoning_budget_msg */ params_base.reasoning_budget_message,1049 /* media_path */ params_base.media_path,1050 /* force_pure_content */ params_base.force_pure_content_parser1051 };1052 }1053 1054 return true;1055 }1056 1057 server_slot * get_slot_by_id(int id_slot) {1058 // note: allow id_slot to be out of bounds (wrap around)1059 id_slot = id_slot % slots.size();1060 1061 for (server_slot & slot : slots) {1062 if (slot.id == id_slot) {1063 return &slot;1064 }1065 }1066 1067 return nullptr;1068 }1069 1070 server_slot * get_available_slot(const server_task & task) {1071 server_slot * ret = nullptr;1072 1073 bool update_cache = false;1074 1075 // find the slot that has at least n% prompt similarity1076 if (ret == nullptr && slot_prompt_similarity != 0.0f) {1077 float sim_best = 0;1078 1079 for (server_slot & slot : slots) {1080 // skip the slot if it is not available1081 if (slot.is_processing()) {1082 continue;1083 }1084 1085 const auto & tokens = slot.prompt.tokens;1086 1087 // skip the slot if it does not contains cached tokens1088 if (tokens.empty()) {1089 continue;1090 }1091 1092 // fraction of the Longest Common Prefix length with respect to the input prompt length1093 const float sim_cur = float(tokens.get_common_prefix(task.tokens)) / task.tokens.size();1094 1095 // select the current slot if the criteria match1096 if (sim_cur > sim_best && sim_cur > slot_prompt_similarity) {1097 sim_best = sim_cur;1098 1099 ret = &slot;1100 }1101 }1102 1103 if (ret != nullptr) {1104 const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size();1105 1106 SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n",1107 sim_best, slot_prompt_similarity, f_keep);1108 1109 // if we are about to lose a large portion of the existing context - save it in the prompt cache1110 if (f_keep < 0.5f) {1111 update_cache = true;1112 }1113 }1114 }1115 1116 // find the slot that has been least recently used1117 if (ret == nullptr) {1118 int64_t t_last = -1;1119 1120 for (server_slot & slot : slots) {1121 // skip the slot if it is not available1122 if (slot.is_processing()) {1123 continue;1124 }1125 1126 // select the current slot if the criteria match1127 if (!ret || slot.t_last_used <= t_last) {1128 t_last = slot.t_last_used;1129 ret = &slot;1130 }1131 }1132 1133 if (ret != nullptr) {1134 SLT_INF(*ret, "selected slot by LRU, t_last = %" PRId64 "\n", t_last);1135 1136 update_cache = true;1137 }1138 }1139 1140 if (ret) {1141 const auto & tokens = ret->prompt.tokens;1142 1143 update_cache = update_cache && prompt_cache;1144 1145 // cache prompts only for completion tasks1146 update_cache = update_cache && task.type == SERVER_TASK_TYPE_COMPLETION;1147 1148 if (update_cache) {1149 SRV_WRN("%s", "updating prompt cache\n");1150 1151 const int64_t t_start = ggml_time_us();1152 1153 // don't save the slot's state if its context is empty1154 if (tokens.size() > 0) {1155 ret->prompt_save(*prompt_cache);1156 }1157 1158 if (!ret->prompt_load(*prompt_cache, task.tokens)) {1159 ret->prompt_clear(false);1160 }1161 1162 prompt_cache->update();1163 1164 SRV_WRN("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);1165 }1166 }1167 1168 return ret;1169 }1170 1171 // return true if at least one slot has been cleared1172 // TODO: improve logic1173 // - smarter decision which slot to clear (LRU or longest prompt?)1174 // - move slot to level 2 cache instead of removing?1175 // - instead of purging, try to store and resume later?1176 bool try_clear_idle_slots() {1177 bool res = false;1178 1179 if (!params_base.kv_unified) {1180 return res;1181 }1182 1183 for (auto & slot : slots) {1184 if (slot.is_processing()) {1185 continue;1186 }1187 1188 if (slot.prompt.n_tokens() > 0) {1189 SRV_WRN("purging slot %d with %zu tokens\n", slot.id, slot.prompt.tokens.size());1190 1191 slot.prompt_clear(false);1192 1193 res = true;1194 1195 // clear slots one by one1196 break;1197 }1198 }1199 1200 return res;