Felipe97/llama-cpp-compiled
01.1k
1#include "server-context.h"2#include "server-chat.h"3#include "server-common.h"4#include "server-http.h"5#include "server-task.h"6#include "server-queue.h"7#include "server-schema.h"8#include "server-stream.h"9 10#include "build-info.h"11#include "common.h"12#include "fit.h"13#include "llama.h"14#include "log.h"15#include "sampling.h"16#include "speculative.h"17#include "mtmd.h"18#include "mtmd-helper.h"19 20#include <algorithm>21#include <cstddef>22#include <cinttypes>23#include <exception>24#include <memory>25#include <filesystem>26#include <random>27#include <utility>28#include <fstream>29 30// fix problem with std::min and std::max31#if defined(_WIN32)32#define WIN32_LEAN_AND_MEAN33#ifndef NOMINMAX34# define NOMINMAX35#endif36#include <windows.h>37#endif38 39constexpr int HTTP_POLLING_SECONDS = 1;40 41static common_speculative_output_limits server_output_limits(const common_params & params) {42 if (params.embedding ||43 (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {44 return { params.n_batch, 1 };45 }46 47 auto result = common_speculative_get_output_limits(48 params.n_batch, params.n_parallel, common_speculative_n_max(¶ms.speculative));49 50 result.total = std::max<int32_t>(1, result.total);51 result.per_seq = std::max<int32_t>(1, result.per_seq);52 return result;53}54 55// synthetic draft verification for benchmarking - accept draft tokens at random instead of by match with the target56// on replay the draft was already accepted before a context checkpoint restore, so repeat the same decisions57static std::vector<llama_token> server_sample_and_accept_synth(58 common_sampler * smpl,59 llama_context * ctx,60 const std::vector<int32_t> & idxs,61 const llama_tokens & draft,62 const std::vector<double> & synth_probs,63 std::mt19937 & rng,64 bool is_replay) {65 GGML_ASSERT(idxs.size() == draft.size() + 1);66 GGML_ASSERT(synth_probs.size() >= draft.size());67 68 std::vector<llama_token> result;69 result.reserve(idxs.size());70 71 const llama_vocab * vocab = llama_model_get_vocab(llama_get_model(ctx));72 std::uniform_real_distribution<double> dist(0.0, 1.0);73 for (size_t i = 0; i < draft.size(); ++i) {74 const llama_token id = common_sampler_sample(smpl, ctx, idxs[i]);75 const bool accept = is_replay || dist(rng) < synth_probs[i];76 // do not accept a drafted EOG token - it would end the generation early77 // on replay the last token is from the target and can be EOG, so skip this check78 if (accept && (is_replay || !llama_vocab_is_eog(vocab, draft[i]))) {79 // synthetic draft tokens do not advance grammar or reasoning state80 // the last replay token is from the target and must advance both81 const bool is_replay_target = is_replay && i + 1 == draft.size();82 common_sampler_accept(smpl, draft[i], is_replay_target);83 result.push_back(draft[i]);84 continue;85 }86 87 common_sampler_accept(smpl, id, true);88 result.push_back(id);89 return result;90 }91 92 const llama_token id = common_sampler_sample(smpl, ctx, idxs[draft.size()]);93 common_sampler_accept(smpl, id, true);94 result.push_back(id);95 96 return result;97}98 99// state diagram: https://github.com/ggml-org/llama.cpp/pull/9283100enum slot_state {101 SLOT_STATE_IDLE,102 SLOT_STATE_WAIT_OTHER, // after assigning a task, but waiting for parent slot to process prompt103 SLOT_STATE_STARTED, // after assigning a task and about to process prompt104 SLOT_STATE_PROCESSING_PROMPT,105 SLOT_STATE_DONE_PROMPT,106 SLOT_STATE_GENERATING,107};108 109struct server_slot; // forward declaration110 111struct server_batch {112 llama_batch batch;113 bool batch_rendered = false;114 115 struct token {116 int32_t id_slot;117 llama_token token;118 llama_pos pos;119 bool output;120 bool is_prompt; // for stats tracking121 };122 std::vector<token> tokens;123 int32_t n_tokens_alloc = 0;124 int32_t n_embd = 0;125 126 // track if given slot can be batched with slots already in the batch127 server_slot * slot_batched = nullptr;128 129 // in embd mode, we temporarily swap out the tokens arr and restore it on clear()130 bool has_embd = false;131 llama_token * tokens_ptr = nullptr;132 std::vector<float> embd;133 134 float alora_scale = -1.0f;135 size_t alora_disabled_id = 0;136 137 server_batch() {138 batch.pos = nullptr; // sentinel: uninitialized batch139 }140 141 ~server_batch() {142 if (batch.pos != nullptr) {143 clear();144 llama_batch_free(batch);145 }146 }147 148 void init(int32_t n_tokens_alloc, int32_t n_embd) {149 this->n_tokens_alloc = n_tokens_alloc;150 this->n_embd = n_embd;151 batch = llama_batch_init(n_tokens_alloc, 0, 1);152 tokens_ptr = batch.token;153 tokens.reserve(n_tokens_alloc);154 }155 156 bool add(int32_t id_slot, llama_token token, llama_pos pos, bool output, bool is_prompt) {157 GGML_ASSERT(!has_embd); // cannot mix tokens + embd in same batch158 GGML_ASSERT(batch.pos != nullptr);159 if ((int32_t)tokens.size() >= n_tokens_alloc) {160 return false;161 }162 tokens.push_back({ id_slot, token, pos, output, is_prompt });163 return true;164 }165 166 bool add(int32_t id_slot, const std::vector<float> & embd_in, llama_pos pos, bool output, bool is_prompt) {167 GGML_ASSERT(batch.pos != nullptr);168 if ((int32_t)tokens.size() >= n_tokens_alloc) {169 return false;170 }171 tokens.push_back({ id_slot, LLAMA_TOKEN_NULL, pos, output, is_prompt });172 has_embd = true;173 embd.insert(embd.end(), embd_in.begin(), embd_in.end());174 return true;175 }176 177 void clear() {178 tokens.clear();179 embd.clear();180 common_batch_clear(batch);181 slot_batched = nullptr;182 alora_scale = -1.0f;183 alora_disabled_id = 0;184 batch_rendered = false;185 has_embd = false;186 if (batch.token == nullptr) {187 batch.token = tokens_ptr;188 batch.embd = nullptr;189 }190 }191 192 int32_t size() const {193 return (int32_t)tokens.size();194 }195 196 void set_output(int32_t idx, bool output) {197 GGML_ASSERT(idx >= 0 && idx < (int32_t)tokens.size());198 tokens[idx].output = output;199 }200 201 void render() {202 GGML_ASSERT(!batch_rendered);203 GGML_ASSERT(batch.pos != nullptr);204 common_batch_clear(batch);205 for (int32_t i = 0; i < size(); i++) {206 const auto & t = tokens[i];207 common_batch_add(batch, t.token, t.pos, { t.id_slot }, t.output);208 }209 if (has_embd) {210 batch.token = nullptr; // will be restored on clear()211 batch.embd = embd.data();212 }213 batch_rendered = true;214 }215 216 llama_batch get_view(int32_t off, int32_t n_tokens) const {217 GGML_ASSERT(batch.pos != nullptr);218 GGML_ASSERT(batch_rendered);219 GGML_ASSERT(off >= 0 && off < size());220 GGML_ASSERT(n_tokens > 0 && off + n_tokens <= size());221 222 auto * token = batch.token ? batch.token + off : nullptr;223 auto * embd = batch.embd ? batch.embd + off * n_embd : nullptr;224 225 llama_batch view = {226 n_tokens,227 token,228 embd,229 batch.pos + off,230 batch.n_seq_id + off,231 batch.seq_id + off,232 batch.logits + off,233 };234 235 return view;236 }237};238 239struct server_slot {240 int id;241 242 llama_context * ctx_tgt = nullptr;243 llama_context * ctx_dft = nullptr;244 245 common_memory mem;246 247 // multimodal248 mtmd_context * mctx = nullptr;249 mtmd::batch_ptr mbatch = nullptr;250 251 // speculative decoding252 common_speculative * spec;253 254 llama_tokens spec_draft;255 llama_tokens spec_prompt;256 std::vector<int32_t> spec_i_batch;257 common_prompt_checkpoint spec_ckpt;258 bool spec_is_replay = false;259 std::mt19937 spec_synth_rng;260 261 // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state262 // see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837263 std::unique_ptr<const server_task> task;264 std::unique_ptr<const server_task> task_prev; // used for debugging265 266 // used to determine the slot that has been used the longest267 int64_t t_last_used = -1;268 269 // generation props270 int32_t n_ctx = 0; // context size per slot271 int32_t n_keep = 0;272 int32_t i_batch = -1;273 274 // effective generation limit for the current task, -1 means unlimited275 int32_t n_predict_max = -1;276 277 size_t last_nl_pos = 0;278 279 std::string generated_text;280 std::string debug_generated_text;281 llama_tokens generated_tokens;282 size_t n_sent_text = 0; // number of sent text character (i.e. handle partial UTF-8 on streaming)283 284 std::vector<completion_token_output> generated_token_probs;285 286 bool has_next_token = true;287 bool has_new_line = false;288 bool truncated = false;289 290 stop_type stop;291 292 std::string stopping_word;293 294 // state295 slot_state state = SLOT_STATE_IDLE;296 297 server_prompt prompt;298 299 bool prompt_save(server_prompt_cache & prompt_cache) const {300 if (prompt.tokens.size() == 0) {301 return false;302 }303 304 const size_t cur_size_tgt = llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE);305 const size_t cur_size_dft = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0;306 307 const size_t cur_size = cur_size_tgt + cur_size_dft;308 309 SRV_TRC(" - saving prompt with length %d, total state size = %.3f MiB (draft: %.3f MiB)\n",310 (int) prompt.tokens.size(), cur_size / (1024.0 * 1024.0), cur_size_dft / (1024.0 * 1024.0));311 312 auto * cur = prompt_cache.alloc(prompt, cur_size_tgt, cur_size_dft);313 if (cur == nullptr) {314 return false;315 }316 317 llama_state_seq_get_data_ext(ctx_tgt, cur->data.main.data(), cur_size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE);318 if (ctx_dft) {319 llama_state_seq_get_data_ext(ctx_dft, cur->data.drft.data(), cur_size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE);320 }321 322 return true;323 }324 325 bool prompt_load(server_prompt_cache & prompt_cache, const server_tokens & tokens) {326 bool res = prompt_cache.load(prompt, tokens, ctx_tgt, ctx_dft, id);327 if (!res) {328 SLT_WRN(*this, "%s", "failed to load prompt from cache\n");329 }330 331 return res;332 }333 334 void prompt_clear() {335 SLT_TRC(*this, "clearing prompt with %zu tokens\n", prompt.tokens.size());336 337 mem.seq_rm(id, -1, -1);338 339 prompt.clear();340 }341 342 std::vector<common_adapter_lora_info> lora;343 int32_t alora_invocation_start = -1;344 345 // sampling346 json json_schema;347 348 common_sampler_ptr smpl;349 350 llama_token sampled; // in speculative mode, this is the last accepted token351 352 // for TTS models, this is the embd generated from prev step, decode this to generate next hidden state353 // corresponding to one token position (size = n_embd)354 std::vector<float> inp_embd;355 356 server_slot_stats stats;357 358 // accepted tokens per draft position359 // not in server_slot_stats to avoid copying to every task result360 std::vector<uint64_t> n_accepted_per_pos;361 362 std::function<void(int /* id_slot */)> callback_on_release;363 std::function<void(const server_slot &)> callback_on_reset; // called before reset()364 365 // this is for printing timings with slot progress, not part of metrics366 int64_t t_print_last = 0;367 int32_t n_gen_last = 0;368 369 void reset() {370 SLT_DBG(*this, "%s", "\n");371 372 spec_is_replay = false;373 374 last_nl_pos = 0;375 generated_text = "";376 has_new_line = false;377 truncated = false;378 stop = STOP_TYPE_NONE;379 stopping_word = "";380 n_sent_text = 0;381 382 if (can_speculate()) {383 spec_draft.clear();384 spec_i_batch.clear();385 spec_ckpt.clear();386 }387 generated_tokens.clear();388 generated_token_probs.clear();389 json_schema = json();390 391 task_prev = std::move(task);392 task.reset();393 394 // note: callback_on_reset() must have run before this, see release()395 stats = {};396 n_accepted_per_pos.clear();397 398 n_predict_max = -1;399 400 llama_set_sampler(ctx_tgt, id, nullptr);401 402 // clear alora start403 alora_invocation_start = -1;404 405 // clear multimodal state406 mbatch.reset();407 }408 409 void init_sampler() const {410 common_sampler_reset(smpl.get());411 412 if (!task->need_sampling()) {413 return;414 }415 416 const int64_t t_start = ggml_time_us();417 418 int n_text = 0;419 420 for (int i = 0; i < (int) prompt.tokens.size(); i++) {421 const llama_token id = prompt.tokens[i];422 423 if (id != LLAMA_TOKEN_NULL) {424 common_sampler_accept(smpl.get(), id, false);425 n_text++;426 }427 }428 429 SLT_TRC(*this, "init sampler, took %0.2f ms, tokens: text = %d, total = %d\n",430 (ggml_time_us() - t_start) / 1000.0, n_text, (int) prompt.tokens.size());431 }432 433 bool need_embd() const {434 GGML_ASSERT(task);435 return task->need_embd();436 }437 438 // if the context does not have a memory module then all embeddings have to be computed within a single ubatch439 // also we cannot split if the pooling would require any past tokens440 // (MTP supports splitting — uses task->need_embd() not need_embd())441 bool can_split() const {442 GGML_ASSERT(task);443 444 return445 !task->need_embd() ||446 (llama_get_memory(ctx_tgt) && llama_pooling_type(ctx_tgt) == LLAMA_POOLING_TYPE_LAST);447 }448 449 bool can_batch_with(server_slot & other_slot) const {450 GGML_ASSERT(task);451 452 return task->type == other_slot.task->type453 && inp_embd.size() == other_slot.inp_embd.size()454 && are_lora_equal(lora, other_slot.lora);455 }456 457 // returns -1 if the generation is limitless458 int32_t n_remaining() const {459 return n_predict_max == -1 ? -1 : n_predict_max - (int32_t) stats.n_gen;460 }461 462 bool has_budget() const {463 return n_predict_max == -1 || n_remaining() > 0;464 }465 466 bool is_processing() const {467 return state != SLOT_STATE_IDLE;468 }469 470 bool can_speculate() const {471 return !!spec;472 }473 474 void add_token(const completion_token_output & token) {475 if (!is_processing()) {476 SLT_WRN(*this, "%s", "slot is not processing\n");477 return;478 }479 480 generated_token_probs.push_back(token);481 }482 483 int get_n_draft_max() const {484 GGML_ASSERT(task);485 486 if (!can_speculate()) {487 return 0;488 }489 490 // determine the max draft that fits the current slot state491 // note: slot.prompt is not yet expanded with the `id` token sampled above492 // also, need to leave space for 1 extra token to allow context shifts493 int n_draft_max = n_ctx - prompt.n_tokens() - 2;494 495 if (n_remaining() > 0) {496 n_draft_max = std::min(n_draft_max, n_remaining() - 1);497 }498 499 SLT_DBG(*this, "max possible draft: %d\n", n_draft_max);500 501 return n_draft_max;502 }503 504 // add sampled token of this slot to the batch, optionally add the speculative draft tokens if any505 void handle_last_sampled_token(server_batch & batch) {506 bool add_ok = true;507 if (spec_draft.empty()) {508 // no speculative decoding509 i_batch = batch.size();510 511 if (!inp_embd.empty()) {512 add_ok &= batch.add(id, inp_embd, prompt.tokens.pos_next(), true, false);513 } else {514 add_ok &= batch.add(id, sampled, prompt.tokens.pos_next(), true, false);515 }516 517 SLT_DBG(*this, "slot decode token, id=%d, n_ctx = %d, n_tokens = %d, truncated = %d\n",518 sampled, n_ctx, prompt.n_tokens(), truncated);519 } else {520 SLT_DBG(*this, "generate_draft: id=%d, #tokens=%zu, #draft=%zu, pos_next=%d\n",521 sampled, prompt.tokens.size(), spec_draft.size(), prompt.tokens.pos_next());522 523 GGML_ASSERT(spec_i_batch.empty());524 525 spec_i_batch.push_back(batch.size());526 for (size_t i = 0; i < spec_draft.size(); i++) {527 spec_i_batch.push_back(batch.size() + i + 1);528 }529 530 auto pos0 = prompt.tokens.pos_next();531 532 add_ok &= batch.add(id, sampled, pos0++, true, false);533 for (auto token : spec_draft) {534 add_ok &= batch.add(this->id, token, pos0++, true, false);535 }536 }537 538 GGML_ASSERT(add_ok && "batch must be large enough to hold the sampled and draft tokens");539 540 prompt.tokens.push_back(sampled);541 prompt.tokens.insert(spec_draft);542 }543 544 void release() {545 if (is_processing()) {546 GGML_ASSERT(task);547 548 SLT_INF(*this, "stop processing: n_tokens = %d, truncated = %d\n", prompt.n_tokens(), truncated);549 550 t_last_used = ggml_time_us();551 552 state = SLOT_STATE_IDLE;553 554 // do not keep context of the child slots - the parent's context is enough555 if (task->is_child()) {556 prompt_clear();557 }558 559 callback_on_reset(*this);560 561 reset();562 563 callback_on_release(id);564 }565 }566 567 size_t find_stopping_strings(const std::string & text, const size_t last_token_size, bool is_full_stop) {568 GGML_ASSERT(task);569 570 size_t stop_pos = std::string::npos;571 572 for (const std::string & word : task->params.antiprompt) {573 size_t pos;574 575 if (is_full_stop) {576 const size_t tmp = word.size() + last_token_size;577 const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;578 579 pos = text.find(word, from_pos);580 } else {581 // otherwise, partial stop582 pos = string_find_partial_stop(text, word);583 }584 585 if (pos != std::string::npos && (stop_pos == std::string::npos || pos < stop_pos)) {586 if (is_full_stop) {587 stop = STOP_TYPE_WORD;588 stopping_word = word;589 has_next_token = false;590 }591 stop_pos = pos;592 }593 }594 595 return stop_pos;596 }597 598 void print_timings_tg() {599 if (stats.n_gen < 100) {600 return;601 }602 603 const int64_t t_now = ggml_time_us();604 605 if (t_now - t_print_last < 3*1000*1000) {606 return;607 }608 609 const double n_gen_second = stats.n_gen_tps();610 const double n_gen_second_win = 1e6 / (t_now - t_print_last) * (stats.n_gen - n_gen_last);611 612 t_print_last = t_now;613 n_gen_last = stats.n_gen;614 615 SLT_INF(*this, "n_gen = %6d, tg = %6.2f t/s, tg_3s = %6.2f t/s\n", (int) stats.n_gen, n_gen_second, n_gen_second_win);616 }617 618 void print_timings_pp() const {619 const double t_prompt_total = stats.t_prompt_ms();620 621 if (t_prompt_total < 3000.0) {622 return;623 }624 625 const double n_prompt_second = stats.n_prompt_tps();626 const double f_progress = task->n_tokens() > 0 ? (double) prompt.n_tokens() / task->n_tokens() : 0.0;627 628 SLT_INF(*this, "prompt processing, n_tokens = %6d, progress = %.2f, t = %6.2f s / %.2f tokens per second\n",629 (int) stats.n_prompt_processed, f_progress, t_prompt_total / 1e3, n_prompt_second);630 }631 632 void print_timings() const {633 const double t_prompt_total = stats.t_prompt_ms();634 const double t_gen_total = stats.t_gen_ms();635 636 const double t_prompt = stats.t_prompt_per_token_ms();637 const double n_prompt_second = stats.n_prompt_tps();638 639 const double t_gen = stats.t_gen_per_token_ms();640 const double n_gen_second = stats.n_gen_tps();641 642 SLT_INF(*this,643 "prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",644 t_prompt_total, (int) stats.n_prompt_processed, t_prompt, n_prompt_second);645 646 SLT_INF(*this,647 " eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",648 t_gen_total, (int) stats.n_gen, t_gen, n_gen_second);649 650 SLT_INF(*this,651 " total time = %10.2f ms / %5d tokens\n",652 t_prompt_total + t_gen_total, (int) (stats.n_prompt_processed + stats.n_gen));653 654 SLT_INF(*this,655 " graphs reused = %10d\n",656 llama_perf_context(ctx_tgt).n_reused);657 658 const int32_t n_draft_total = stats.n_draft_tokens;659 const int32_t n_draft_accepted = stats.n_draft_accepted;660 const int32_t n_draft_verif_steps = stats.n_draft_verif_steps;661 662 if (n_draft_total > 0) {663 const float draft_ratio = (float) n_draft_accepted / n_draft_total;664 const double mean_acc_len = n_draft_verif_steps > 0 ? 1.0 + (double) n_draft_accepted / (double) n_draft_verif_steps : 1.0;665 666 std::string acceptance_rates_per_pos;667 if (n_draft_verif_steps > 0) {668 for (size_t i = 0; i < n_accepted_per_pos.size(); ++i) {669 if (i > 0) {670 acceptance_rates_per_pos += ", ";671 }672 acceptance_rates_per_pos += string_format("%.3f", (double) n_accepted_per_pos[i] / (double) n_draft_verif_steps);673 }674 }675 676 SLT_INF(*this,677 "draft acceptance = %0.5f (%5d accepted / %5d generated), mean len = %5.2f\n",678 draft_ratio, n_draft_accepted, n_draft_total, mean_acc_len);679 SLT_TRC(*this,680 " acc per pos = (%s)\n", acceptance_rates_per_pos.c_str());681 }682 683 common_speculative_print_stats(spec);684 }685 686 json to_json(bool only_metrics = false) const {687 json res;688 689 res = {690 {"id", id},691 {"n_ctx", n_ctx},692 {"speculative", can_speculate()},693 {"is_processing", is_processing()},694 };695 696 const auto & ptask = task ? task : task_prev;697 698 if (ptask) {699 res["id_task"] = ptask->id;700 res["n_prompt_tokens"] = (int32_t) prompt.tokens.size();701 res["n_prompt_tokens_processed"] = stats.n_prompt_processed;702 res["n_prompt_tokens_cache"] = stats.n_prompt_cached;703 res["params"] = ptask->params.to_json(only_metrics);704 res["next_token"] = json::array({705 {706 {"has_next_token", has_next_token},707 {"has_new_line", has_new_line},708 {"n_remain", n_remaining()},709 {"n_decoded", stats.n_gen},710 }711 });712 713 if (!only_metrics) {714 res["prompt"] = ptask->tokens.detokenize(ctx_tgt, true);715 res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;716 }717 }718 719 return res;720 }721 722 void copy_state_to(server_slot & other) const {723 GGML_ASSERT(state == SLOT_STATE_DONE_PROMPT);724 725 mem.seq_rm(other.id, -1, -1);726 mem.seq_cp(id, other.id, -1, -1);727 728 other.i_batch = i_batch;729 730 other.stats = stats;731 732 other.prompt = prompt.clone();733 other.init_sampler();734 }735};736 737// returns 0 on success738// caller need to update prompt.tokens after a successful call to keep track of the processing progress739// note: this is not a member of server_slot because we want to run it inside yield_to_queue740// slot is passed as const to avoid accidental modification of the slot state741// some pointers are allowed to be used, they are not used by to_json()742static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch, size_t idx, size_t & n_tokens_out) {743 GGML_ASSERT(slot.mctx);744 const auto & mctx = slot.mctx;745 const auto & input_tokens = slot.task->tokens;746 const auto & chunk = input_tokens.find_chunk(idx);747 int32_t res = 0;748 749 auto try_decode = [&]() -> int32_t {750 if (mbatch) {751 float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get());752 if (embd) {753 void * cb_data = slot.spec;754 static auto cb = [](llama_batch batch, void * user_data) {755 common_speculative * spec = static_cast<common_speculative *>(user_data);756 if (!common_speculative_process(spec, batch)) {757 return 1;758 }759 return 0;760 };761 762 llama_pos new_n_past; // unused for now763 res = mtmd_helper_decode_image_chunk(764 mctx,765 slot.ctx_tgt,766 chunk.get(),767 embd,768 slot.prompt.tokens.pos_next(),769 slot.id,770 llama_n_batch(slot.ctx_tgt),771 &new_n_past,772 cb,773 cb_data774 );775 if (res != 0) {776 SLT_ERR(slot, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res);777 return -1;778 }779 n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get());780 return 0; // success781 }782 }783 return 1; // (non-error) need to create & encode batch784 };785 786 // if the batch is already exist, try searching & encode787 res = try_decode();788 if (res == 0) {789 return 0;790 }791 if (res < 0) {792 // fatal error793 return res;794 }795 796 // otherwise, the batch is either uninitialized or is used up797 // we need to create & encode a new batch798 mbatch.reset(mtmd_batch_init(mctx));799 res = mtmd_batch_add_chunk(mbatch.get(), chunk.get());800 GGML_ASSERT(res == 0); // we should never have an empty batch801 802 // try batching as much as possible803 int n_added = 1;804 size_t idx_cur = idx;805 while (res == 0) {806 auto [next_chunk, next_idx] = input_tokens.find_next_media_chunk(idx_cur);807 if (next_chunk == nullptr) {808 break;809 }810 res = mtmd_batch_add_chunk(mbatch.get(), next_chunk->get());811 n_added += (res == 0 ? 1 : 0);812 idx_cur = next_idx;813 SLT_DBG(slot, "try adding media chunk idx = %zu to batch, res = %d\n", next_idx, res);814 // if res != 0, batch is full or chunk is not compatible -> this loop breaks815 }816 817 // TODO @ngxson : move this log line to debug when it become more stable818 SLT_TRC(slot, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added);819 820 res = mtmd_batch_encode(mbatch.get());821 if (res != 0) {822 SLT_ERR(slot, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res);823 return -1;824 }825 826 return try_decode();827}828 829//830// server_context_impl (private implementation)831//832 833struct server_context_impl {834 friend struct server_context;835 836public:837 // only use these pointers outside of this class:838 // - when not in sleeping state839 // - and, with thread-safe APIs (e.g., tokenizer calls)840 llama_model * model_tgt = nullptr;841 842 mtmd_context * mctx = nullptr;843 // note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct844 mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();845 const llama_vocab * vocab = nullptr;846 847 server_queue queue_tasks;848 server_response queue_results;849 850 // note: chat_params must not be refreshed upon existing sleeping state851 server_chat_params chat_params;852 853 server_state_callback_t callback_state = [](server_state, json) -> void {};854 855 server_context_impl() {856 mtmd_helper_log_set(common_log_default_callback, nullptr);857 }858 859 ~server_context_impl() {860 if (!sleeping) {861 // destroy() is already called when entering sleeping state862 // we don't call it again here to avoid double free863 destroy();864 }865 }866 867 server_metrics get_metrics() const {868 return metrics;869 }870 871 void reset_metrics_bucket() {872 metrics.reset_bucket();873 }874 875private:876 // note: accessing these fields outside of this class is not thread-safe877 // use server_context methods instead878 879 common_params params_base;880 881 // note: keep these alive - they determine the lifetime of the model, context, etc.882 common_init_result_ptr llama_init;883 884 llama_context * ctx_tgt = nullptr;885 886 server_batch batch;887 888 llama_model * model_dft = nullptr;889 llama_context * ctx_dft = nullptr;890 891 common_speculative_init_result_ptr spec_init;892 893 common_context_seq_rm_type ctx_tgt_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;894 common_context_seq_rm_type ctx_dft_seq_rm_type = COMMON_CONTEXT_SEQ_RM_TYPE_NO;895 896 common_speculative_ptr spec;897 898 bool add_bos_token = true;899 900 int32_t n_ctx; // total context for all clients / slots901 902 // set to llama_model_n_swa(model)903 // if swa_full is enabled, this is set to 0 to simulate a non-SWA model904 int32_t n_swa;905 906 // slots / clients907 std::vector<server_slot> slots;908 909 int trace = 0; // env: LLAMA_TRACE910 int slots_debug = 0; // env: LLAMA_SERVER_SLOTS_DEBUG911 int slots_n_diff = 0; // env: LLAMA_SERVER_SLOTS_N_DIFF912 913 int n_empty_consecutive = 0;914 915 std::unique_ptr<server_prompt_cache> prompt_cache;916 917 server_metrics metrics;918 919 // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync920 // note: kept out of server_metrics, which is copied as-is into the task result921 int64_t t_decode_start = 0; // start of the last submitted decode922 int64_t t_prompt_start = 0; // start of the oldest queued prompt decode923 uint64_t n_prompt_queued = 0;924 925 json json_ui_settings = json::object();926 927 // Necessary similarity of prompt for slot selection928 float slot_prompt_similarity = 0.0f;929 930 std::string model_name; // name of the loaded model, to be used by API931 std::set<std::string> model_aliases; // additional names for the model932 std::set<std::string> model_tags; // informational tags933 934 bool sleeping = false;935 936 int64_t t_last_load_progress_ms = 0;937 938 void destroy() {939 spec.reset();940 spec_init.reset();941 942 ctx_dft = nullptr;943 model_dft = nullptr;944 945 llama_init.reset();946 947 ctx_tgt = nullptr;948 model_tgt = nullptr;949 950 mtmd_free(mctx);951 mctx = nullptr;952 }953 954 void handle_sleeping_state(bool new_state) {955 GGML_ASSERT(sleeping != new_state);956 if (new_state) {957 if (callback_state) {958 callback_state(SERVER_STATE_SLEEPING, {});959 // note: for sleeping == false, event is emitted by load_model()960 }961 SRV_INF("%s", "server is entering sleeping state\n");962 destroy();963 } else {964 SRV_INF("%s", "server is exiting sleeping state\n");965 if (!load_model(params_base)) {966 GGML_ABORT("failed to reload model after sleeping");967 }968 }969 sleeping = new_state;970 }971 972 struct load_progress_data {973 server_context_impl * ctx;974 std::string stage;975 std::vector<std::string> stages;976 int64_t t_last_load_progress_ms = 0;977 load_progress_data(server_context_impl * ctx, const std::string & stage) : ctx(ctx), stage(stage) {}978 };979 static bool load_progress_callback(float progress, void * user_data) {980 auto * d = static_cast<load_progress_data *>(user_data);981 GGML_ASSERT(d);982 // always emit the first and final sample; throttle the rest to one per 200ms983 {984 auto & t_last = d->t_last_load_progress_ms;985 const int64_t t_now = ggml_time_ms();986 const bool first = t_last == 0;987 const bool done = progress >= 1.0f;988 const bool throttled = !first && !done && (t_now - t_last) < 200;989 if (throttled) {990 return true;991 }992 t_last = t_now;993 }994 if (d->ctx->callback_state) {995 d->ctx->callback_state(SERVER_STATE_LOADING, {996 {"stages", d->stages},997 {"current", d->stage},998 {"value", progress},999 });1000 }1001 return true;1002 }1003 1004 // load the model and initialize llama_context1005 // this may also be called to resume from sleeping state1006 bool load_model(common_params & params) {1007 load_progress_data load_progress_text (this, "text_model");1008 load_progress_data load_progress_mmproj(this, "mmproj_model");1009 load_progress_data load_progress_spec (this, "spec_model");1010 1011 const bool is_resume = sleeping;1012 1013 params_base = params;1014 const auto output_limits = server_output_limits(params_base);1015 params_base.n_outputs_max = output_limits.total;1016 params_base.n_outputs_max_per_seq = output_limits.per_seq;1017 1018 const bool has_mmproj = !params.mmproj.path.empty();1019 const bool has_draft = params.speculative.has_dft();1020 const bool spec_mtp = std::find(params_base.speculative.types.begin(),1021 params_base.speculative.types.end(),1022 COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end();1023 const bool has_spec = has_draft || spec_mtp;1024 1025 if (callback_state) {1026 std::vector<std::string> stages = {"text_model"};1027 if (has_spec) {1028 stages.push_back("spec_model");1029 }1030 if (has_mmproj) {1031 stages.push_back("mmproj_model");1032 }1033 load_progress_text.stages = stages;1034 load_progress_mmproj.stages = stages;1035 load_progress_spec.stages = stages;1036 1037 // trigger 0% progress1038 load_progress_callback(0.0f, &load_progress_text);1039 }1040 1041 1042 SRV_INF("loading model '%s'\n", params.model.get_name().c_str());1043 SRV_TRC("local path '%s'\n", params.model.path.c_str());1044 1045 std::string & mmproj_path = params_base.mmproj.path;1046 mtmd_context_params mparams = mtmd_context_params_default();1047 if (has_mmproj) {1048 mparams.use_gpu = params_base.mmproj_use_gpu;1049 mparams.device = params_base.mmproj_device;1050 mparams.print_timings = false;1051 mparams.n_threads = params_base.cpuparams.n_threads;1052 mparams.flash_attn_type = params_base.flash_attn_type;1053 mparams.warmup = params_base.warmup;1054 mparams.image_min_tokens = params_base.image_min_tokens;1055 mparams.image_max_tokens = params_base.image_max_tokens;1056 mparams.batch_max_tokens = params_base.mtmd_batch_max_tokens;1057 mparams.media_marker = get_media_marker();1058 // progress callback1059 mparams.progress_callback = load_progress_callback;1060 mparams.progress_callback_user_data = &load_progress_mmproj;1061 }1062 1063 // optionally get the memory usage of mmproj1064 if (has_mmproj && params_base.fit_params) {1065 int64_t t_start = ggml_time_us();1066 auto mmproj_mem = mtmd_get_memory_usage(mmproj_path.c_str(), mparams);1067 int64_t t_elapsed = ggml_time_us() - t_start;1068 if (!mmproj_mem.empty()) {1069 size_t total = 0;1070 for (auto & [dev, size] : mmproj_mem) {1071 total += size;1072 }1073 SRV_TRC("[mtmd] estimated worst-case memory usage of mmproj is %.2f MiB (took %.2f ms)\n", total / (1024.0 * 1024.0), t_elapsed / 1000.0);1074 GGML_ASSERT(!params_base.fit_params_target.empty());1075 for (auto & [dev, size] : mmproj_mem) {1076 for (size_t i = 0; i < ggml_backend_dev_count(); i++) {1077 if (ggml_backend_dev_get(i) == dev) {1078 if (i < params_base.fit_params_target.size()) {1079 SRV_DBG("[mtmd] adding %.2f MiB to fit_params_target for device %s\n", size / (1024.0 * 1024.0), ggml_backend_dev_name(dev));1080 params_base.fit_params_target[i] += size;1081 }1082 break;1083 }1084 }1085 }1086 } else {1087 SRV_ERR("%s", "[mtmd] failed to get memory usage of mmproj\n");1088 }1089 }1090 1091 // note: the draft / MTP context is fitted together with the target model, see common_fit_extra_model1092 1093 // attach a progress callback1094 {1095 params_base.load_progress_callback = load_progress_callback;1096 params_base.load_progress_callback_user_data = &load_progress_text;1097 }1098 1099 llama_init = common_init_from_params(params_base);1100 1101 model_tgt = llama_init->model();1102 ctx_tgt = llama_init->context();1103 1104 if (model_tgt == nullptr) {1105 SRV_ERR("failed to load model, '%s'\n", params_base.model.path.c_str());1106 return false;1107 }1108 1109 if (ctx_tgt == nullptr) {1110 SRV_ERR("failed to create_context with model '%s'\n", params_base.model.path.c_str());1111 return false;1112 }1113 1114 vocab = llama_model_get_vocab(model_tgt);1115 1116 n_ctx = llama_n_ctx(ctx_tgt);1117 1118 add_bos_token = llama_vocab_get_add_bos(vocab);1119 1120 if (has_spec) {1121 // spec_mtp doesn't use load a model internally, so we report 0.0 and 1.0 manually1122 load_progress_callback(0.0f, &load_progress_spec);1123 load_progress_spec.t_last_load_progress_ms = 0; // reset so internal cbs aren't delayed1124 1125 {1126 common_params params_dft = common_base_params_to_speculative(params_base);1127 1128 // progress callback1129 params_dft.load_progress_callback = load_progress_callback;1130 params_dft.load_progress_callback_user_data = &load_progress_spec;1131 1132 spec_init = common_speculative_init_from_params(params_dft, model_tgt, ctx_tgt);1133 model_dft = spec_init->model();1134 ctx_dft = spec_init->context();1135 1136 if (has_draft && model_dft == nullptr) {1137 SRV_ERR("failed to load draft model, '%s'\n", params_dft.model.path.c_str());1138 return false;1139 }1140 1141 if (ctx_dft == nullptr) {1142 SRV_ERR("%s", "failed to create MTP context\n");1143 return false;1144 }1145 1146 params_base.speculative.draft.ctx_tgt = ctx_tgt;1147 params_base.speculative.draft.ctx_dft = ctx_dft;1148 }1149 1150 load_progress_callback(1.0f, &load_progress_spec);1151 }1152 1153 if (has_mmproj) {1154 if (callback_state) {1155 callback_state(SERVER_STATE_LOADING, {{"stage", "mmproj_model"}});1156 }1157 1158 if (!is_resume) {1159 mtmd_helper_log_set(common_log_default_callback, nullptr);1160 }1161 1162 mctx = mtmd_init_from_file(mmproj_path.c_str(), model_tgt, mparams);1163 if (mctx == nullptr) {1164 SRV_ERR("failed to load multimodal model, '%s'\n", mmproj_path.c_str());1165 return false;1166 }1167 SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str());1168 1169 init_opt.video_params.fps_target = params_base.video_fps;1170 init_opt.video_params.timestamp_interval_ms = params_base.video_timestamp_interval_ms;1171 init_opt.video_params.ffmpeg_bin_dir = params_base.video_ffmpeg_bin_dir.empty()1172 ? nullptr : params_base.video_ffmpeg_bin_dir.c_str();1173 1174 if (params_base.ctx_shift) {1175 params_base.ctx_shift = false;1176 SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");1177 }1178 1179 if (params_base.n_cache_reuse) {1180 params_base.n_cache_reuse = 0;1181 SRV_WRN("%s\n", "cache_reuse is not supported by multimodal, it will be disabled");1182 }1183 }1184 1185 if (!llama_memory_can_shift(llama_get_memory(ctx_tgt))) {1186 if (params_base.ctx_shift) {1187 params_base.ctx_shift = false;1188 SRV_WRN("%s\n", "ctx_shift is not supported by this context, it will be disabled");1189 }1190 1191 if (params_base.n_cache_reuse) {1192 params_base.n_cache_reuse = 0;1193 SRV_WRN("%s\n", "cache_reuse is not supported by this context, it will be disabled");1194 }1195 }1196 1197 if (llama_model_n_swa(model_tgt) == 0) {1198 if (params_base.swa_full) {1199 params_base.swa_full = false;1200 SRV_WRN("%s\n", "swa_full is not supported by this model, it will be disabled");