echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0762
1#include "common.h"2#include "download.h"3#include "log.h"4#include "llama.h"5#include "mtmd.h"6#include "mtmd-helper.h"7#include "chat.h"8#include "base64.hpp"9 10#include "server-common.h"11 12#include <random>13#include <sstream>14#include <fstream>15 16json format_error_response(const std::string & message, const enum error_type type) {17 std::string type_str;18 int code = 500;19 switch (type) {20 case ERROR_TYPE_INVALID_REQUEST:21 type_str = "invalid_request_error";22 code = 400;23 break;24 case ERROR_TYPE_AUTHENTICATION:25 type_str = "authentication_error";26 code = 401;27 break;28 case ERROR_TYPE_NOT_FOUND:29 type_str = "not_found_error";30 code = 404;31 break;32 case ERROR_TYPE_SERVER:33 type_str = "server_error";34 code = 500;35 break;36 case ERROR_TYPE_PERMISSION:37 type_str = "permission_error";38 code = 403;39 break;40 case ERROR_TYPE_NOT_SUPPORTED:41 type_str = "not_supported_error";42 code = 501;43 break;44 case ERROR_TYPE_UNAVAILABLE:45 type_str = "unavailable_error";46 code = 503;47 break;48 case ERROR_TYPE_EXCEED_CONTEXT_SIZE:49 type_str = "exceed_context_size_error";50 code = 400;51 break;52 }53 return json {54 {"code", code},55 {"message", message},56 {"type", type_str},57 };58}59 60//61// random string / id62//63 64std::string random_string() {65 static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");66 67 std::random_device rd;68 std::mt19937 generator(rd());69 70 std::string result(32, ' ');71 72 for (int i = 0; i < 32; ++i) {73 result[i] = str[generator() % str.size()];74 }75 76 return result;77}78 79std::string gen_chatcmplid() {80 return "chatcmpl-" + random_string();81}82 83std::string gen_tool_call_id() {84 return random_string();85}86 87const char * get_media_marker() {88 static const std::string marker = []() {89 // allow user to pin a reproducible marker via env var90 const char * env = getenv("LLAMA_MEDIA_MARKER");91 if (env && env[0] != '\0') {92 return std::string(env);93 }94 return std::string("<__media_") + random_string() + "__>";95 }();96 return marker.c_str();97}98 99//100// lora utils101//102 103bool lora_all_alora(const std::vector<common_adapter_lora_info> & loras) {104 bool found_alora = false;105 for (const auto & lora : loras) {106 if (lora.scale != 0) {107 if (llama_adapter_get_alora_n_invocation_tokens(lora.ptr) == 0) {108 return false;109 }110 found_alora = true;111 }112 }113 return found_alora;114}115 116bool lora_should_clear_cache(117 const std::vector<common_adapter_lora_info> & current,118 const std::vector<common_adapter_lora_info> & next) {119 120 // This should always be called after determining that the two sets are121 // _not_ equal. This assert is therefore some slightly wasted work and122 // should be safe to remove as long as this method is called correctly.123 GGML_ASSERT(!are_lora_equal(current, next));124 125 return (126 !(lora_get_enabled_ids(current).empty() || lora_all_alora(current)) ||127 !lora_all_alora(next));128}129 130std::map<int, float> parse_lora_request(const json & data) {131 std::map<int, float> lora;132 133 // set value134 for (const auto & entry : data) {135 int id = json_value(entry, "id", -1);136 float scale = json_value(entry, "scale", 0.0f);137 lora[id] = scale;138 }139 140 return lora;141}142 143bool are_lora_equal(144 const std::vector<common_adapter_lora_info> & l1,145 const std::vector<common_adapter_lora_info> & l2) {146 if (l1.size() != l2.size()) {147 return false;148 }149 for (size_t i = 0; i < l1.size(); ++i) {150 // we don't check lora.path to reduce the time complexity151 if (l1[i].scale != l2[i].scale || l1[i].ptr != l2[i].ptr) {152 return false;153 }154 }155 return true;156}157 158std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras) {159 std::vector<size_t> enabled_ids;160 for (size_t i = 0; i < loras.size(); ++i) {161 if (loras[i].scale > 0) {162 enabled_ids.push_back(i);163 }164 }165 return enabled_ids;166}167 168//169// base64 utils (TODO: use the base64::decode from base64.hpp)170//171 172static const std::string base64_chars =173 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"174 "abcdefghijklmnopqrstuvwxyz"175 "0123456789+/";176 177static inline bool is_base64(uint8_t c) {178 return (isalnum(c) || (c == '+') || (c == '/'));179}180 181static inline raw_buffer base64_decode(const std::string & encoded_string) {182 int i = 0;183 int j = 0;184 int in_ = 0;185 186 int in_len = encoded_string.size();187 188 uint8_t char_array_4[4];189 uint8_t char_array_3[3];190 191 raw_buffer ret;192 193 while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {194 char_array_4[i++] = encoded_string[in_]; in_++;195 if (i == 4) {196 for (i = 0; i < 4; i++) {197 char_array_4[i] = base64_chars.find(char_array_4[i]);198 }199 200 char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);201 char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);202 char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];203 204 for (i = 0; (i < 3); i++) {205 ret.push_back(char_array_3[i]);206 }207 208 i = 0;209 }210 }211 212 if (i) {213 for (j = i; j < 4; j++) {214 char_array_4[j] = 0;215 }216 217 for (j = 0; j < 4; j++) {218 char_array_4[j] = base64_chars.find(char_array_4[j]);219 }220 221 char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);222 char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);223 char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];224 225 for (j = 0; j < i - 1; j++) {226 ret.push_back(char_array_3[j]);227 }228 }229 230 return ret;231}232 233//234// server_tokens implementation235//236 237server_tokens::server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd) : has_mtmd(has_mtmd) {238 for (size_t i = 0; i < mtmd_chunks.size(); ++i) {239 push_back(mtmd_chunks[i]);240 }241}242 243server_tokens::server_tokens(const llama_tokens & tokens, bool has_mtmd) : has_mtmd(has_mtmd), tokens(tokens) {244}245 246llama_pos server_tokens::pos_next(int64_t n_tokens) const {247 if (!has_mtmd) {248 if (n_tokens < 0) {249 return tokens.size();250 }251 252 return n_tokens;253 }254 255 if (n_tokens < 0) {256 llama_pos res = tokens.size();257 258 for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {259 const auto & chunk = it->second;260 res += mtmd_input_chunk_get_n_pos(chunk.get()) - mtmd_input_chunk_get_n_tokens(chunk.get());261 }262 263 return res;264 }265 266 int64_t idx = 0;267 llama_pos pos = 0;268 269 GGML_ASSERT(n_tokens <= (int64_t)tokens.size());270 271 while (idx < n_tokens) {272 const auto media_it = map_idx_to_media.find(idx);273 if (media_it != map_idx_to_media.end()) {274 const auto & chunk = media_it->second;275 const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());276 const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());277 278 pos += n_pos;279 idx += n_tok;280 } else {281 pos++;282 idx++;283 }284 }285 286 return pos;287}288 289size_t server_tokens::size_up_to_pos(llama_pos max_pos) const {290 if (!has_mtmd) {291 return std::min((size_t)max_pos, tokens.size());292 }293 294 size_t idx = 0;295 llama_pos pos = 0;296 297 while (idx < tokens.size()) {298 const auto media_it = map_idx_to_media.find(idx);299 if (media_it != map_idx_to_media.end()) {300 const auto & chunk = media_it->second;301 const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());302 const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());303 304 pos += n_pos;305 idx += n_tok;306 } else {307 pos++;308 idx++;309 }310 311 if (pos >= max_pos) {312 break;313 }314 }315 316 return idx;317}318 319std::string server_tokens::str() const {320 std::ostringstream oss;321 oss << "tokens: ";322 for (size_t idx = 0; idx < tokens.size(); ++idx) {323 llama_token t = tokens[idx];324 oss << "idx:" << idx << " ";325 if (t == LLAMA_TOKEN_NULL) {326 oss << "<embd> ";327 } else {328 oss << t << " ";329 }330 }331 oss << "\n";332 oss << "image idx: ";333 for (const auto & it : map_idx_to_media) {334 oss << it.first << ", ";335 }336 return oss.str();337}338 339const mtmd::input_chunk_ptr & server_tokens::find_chunk(size_t idx) const {340 auto it = map_idx_to_media.find(idx);341 if (it != map_idx_to_media.end()) {342 return it->second;343 }344 throw std::runtime_error("Chunk not found");345}346 347void server_tokens::push_back(llama_token tok) {348 if (tok == LLAMA_TOKEN_NULL) {349 throw std::runtime_error("Invalid token");350 }351 tokens.emplace_back(tok);352}353 354void server_tokens::push_back(const mtmd_input_chunk * chunk) {355 auto type = mtmd_input_chunk_get_type(chunk);356 if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {357 GGML_ASSERT(has_mtmd);358 const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);359 size_t start_idx = tokens.size();360 for (size_t i = 0; i < n_tokens; ++i) {361 tokens.emplace_back(LLAMA_TOKEN_NULL);362 }363 mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_copy(chunk));364 map_idx_to_media[start_idx] = std::move(new_chunk);365 } else if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) {366 size_t n_tokens;367 const auto * text_tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);368 for (size_t i = 0; i < n_tokens; ++i) {369 push_back(text_tokens[i]);370 }371 } else {372 GGML_ABORT("Invalid chunk type");373 }374}375 376void server_tokens::push_back(server_tokens & tokens) {377 size_t start_idx = size();378 for (size_t i = 0; i < tokens.size(); i++) {379 push_back(tokens[i]);380 }381 if (tokens.has_mtmd) {382 // Assert if we are copying MTMD chunks to a server_tokens that does not have mtmd.383 // We could also just check, but this will prevent silently dropping MTMD data.384 GGML_ASSERT(has_mtmd);385 for (auto it = tokens.map_idx_to_media.begin(); it != tokens.map_idx_to_media.end(); ) {386 auto * chunk = tokens.map_idx_to_media[it->first].get();387 mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_copy(chunk));388 map_idx_to_media[start_idx + it->first] = std::move(new_chunk);389 }390 }391}392 393void server_tokens::insert(const llama_tokens & inp_tokens) {394 tokens.insert(tokens.end(), inp_tokens.begin(), inp_tokens.end());395}396 397const llama_tokens & server_tokens::get_tokens() const {398 GGML_ASSERT(!has_mtmd);399 return tokens;400}401 402llama_tokens server_tokens::get_text_tokens() const {403 llama_tokens res;404 res.reserve(tokens.size());405 for (llama_token t : tokens) {406 if (t != LLAMA_TOKEN_NULL) {407 res.push_back(t);408 }409 }410 return res;411}412 413void server_tokens::set_token(llama_pos pos, llama_token id) {414 GGML_ASSERT(!has_mtmd); // only allow this if mtmd is disabled415 tokens[pos] = id;416}417 418void server_tokens::keep_first(size_t n) {419 GGML_ASSERT(n <= tokens.size());420 if (has_mtmd) {421 if (n == tokens.size()) {422 return; // nothing to do423 }424 // we throw an error if we try to remove a token in the middle of an image425 // for ex. with input of 5 text tokens and 2 images:426 // [0] [1] [2] [3] [4] [img0] [img0] [img0] [img1] [img1]427 // n 1 2 3 4 5 6 7 8 9 10428 // allowed to resize ^ ^429 // disallowed to resize ^ ^ ^430 if (n > 0) {431 // make sure we never remove tokens in the middle of an image432 // note that the case where we keep a full image at the end is allowed:433 // tokens[n - 1] == LLAMA_TOKEN_NULL && tokens[n] != LLAMA_TOKEN_NULL434 if (tokens[n - 1] == LLAMA_TOKEN_NULL && tokens[n] == LLAMA_TOKEN_NULL) {435 find_chunk(n - 1); // will throw an error if the token is not begin-of-chunk436 }437 }438 // remove all image chunks that are not used anymore439 for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ) {440 size_t idx = it->first;441 if (idx >= n) {442 it = map_idx_to_media.erase(it);443 } else {444 ++it;445 }446 }447 }448 tokens.resize(n);449}450 451std::string server_tokens::detokenize(const llama_context * ctx, bool special) const {452 llama_tokens text_tokens;453 text_tokens.reserve(tokens.size());454 for (const auto & t : tokens) {455 if (t != LLAMA_TOKEN_NULL) {456 text_tokens.push_back(t);457 }458 }459 return common_detokenize(ctx, text_tokens, special);460}461 462size_t server_tokens::get_common_prefix(const server_tokens & b) const {463 const size_t max_idx = std::min(tokens.size(), b.tokens.size());464 465 if (!has_mtmd) {466 for (size_t i = 0; i < max_idx; ++i) {467 if (tokens[i] == b.tokens[i]) {468 continue;469 }470 471 return i;472 }473 474 return max_idx;475 }476 477 for (size_t i = 0; i < max_idx; ++i) {478 const llama_token ai = tokens[i];479 const llama_token bi = b.tokens[i];480 481 if (ai == LLAMA_TOKEN_NULL && bi == LLAMA_TOKEN_NULL) {482 const auto & a_chunk = find_chunk(i);483 const auto & b_chunk = b.find_chunk(i);484 485 GGML_ASSERT(a_chunk && b_chunk);486 487 const std::string id_ai = mtmd_input_chunk_get_id(a_chunk.get());488 const std::string id_bi = mtmd_input_chunk_get_id(b_chunk.get());489 490 const size_t n_tok_a = mtmd_input_chunk_get_n_tokens(a_chunk.get());491 const size_t n_tok_b = mtmd_input_chunk_get_n_tokens(b_chunk.get());492 493 if (id_ai == id_bi && n_tok_a == n_tok_b) {494 GGML_ASSERT(n_tok_a > 0 && "Invalid media chunk"); // should never happen495 i += n_tok_a - 1; // will be +1 by the for loop496 continue;497 }498 499 return i;500 }501 502 if (ai == bi) {503 continue;504 }505 506 return i;507 }508 509 return max_idx; // all tokens are equal510}511 512bool server_tokens::validate(const struct llama_context * ctx) const {513 const llama_model * model = llama_get_model(ctx);514 const llama_vocab * vocab = llama_model_get_vocab(model);515 const int32_t n_vocab = llama_vocab_n_tokens(vocab);516 517 for (size_t i = 0; i < tokens.size(); ++i) {518 const auto & t = tokens[i];519 if (t == LLAMA_TOKEN_NULL) {520 try {521 const auto & chunk = find_chunk(i);522 size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk.get());523 i += n_tokens - 1; // will be +1 by the for loop524 } catch (const std::exception & e) {525 return false;526 }527 } else if (t < 0 || t >= n_vocab) {528 return false;529 }530 }531 return true;532}533 534int32_t server_tokens::process_chunk(535 llama_context * ctx,536 mtmd_context * mctx,537 size_t idx,538 llama_pos pos,539 int32_t seq_id,540 size_t & n_tokens_out) const {541 const auto & chunk = find_chunk(idx);542 const char * name = mtmd_input_chunk_get_type(chunk.get()) == MTMD_INPUT_CHUNK_TYPE_IMAGE543 ? "image" : "audio";544 SRV_INF("processing %s...\n", name);545 int32_t n_batch = llama_n_batch(ctx);546 int64_t t0 = ggml_time_ms();547 llama_pos new_n_past; // unused for now548 int32_t result = mtmd_helper_eval_chunk_single(mctx, ctx,549 chunk.get(),550 pos,551 seq_id,552 n_batch,553 true, // logits last554 &new_n_past);555 SRV_INF("%s processed in %" PRId64 " ms\n", name, ggml_time_ms() - t0);556 if (result != 0) {557 LOG_ERR("mtmd_helper_eval failed with status %d", result);558 n_tokens_out = 0;559 return result;560 }561 n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get());562 return 0;563}564 565server_tokens server_tokens::clone() const {566 server_tokens res;567 res.has_mtmd = has_mtmd;568 res.tokens = tokens;569 for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {570 size_t idx = it->first;571 const mtmd::input_chunk_ptr & chunk = it->second;572 res.map_idx_to_media[idx] = mtmd::input_chunk_ptr(mtmd_input_chunk_copy(chunk.get()));573 }574 return res;575}576 577//578// tokenizer and input processing utils579//580 581bool json_is_array_of_numbers(const json & data) {582 if (data.is_array()) {583 for (const auto & e : data) {584 if (!e.is_number_integer()) {585 return false;586 }587 }588 return true;589 }590 return false;591}592 593bool json_is_array_of_mixed_numbers_strings(const json & data) {594 bool seen_string = false;595 bool seen_number = false;596 if (data.is_array()) {597 for (const auto & e : data) {598 seen_string |= e.is_string();599 seen_number |= e.is_number_integer();600 if (seen_number && seen_string) {601 return true;602 }603 }604 }605 return false;606}607 608bool json_is_array_and_contains_numbers(const json & data) {609 if (data.is_array()) {610 for (const auto & e : data) {611 if (e.is_number_integer()) {612 return true;613 }614 }615 return false;616 }617 return false;618}619 620json json_get_nested_values(const std::vector<std::string> & paths, const json & js) {621 json result = json::object();622 623 for (const std::string & path : paths) {624 json current = js;625 const auto keys = string_split<std::string>(path, /*separator*/ '/');626 bool valid_path = true;627 for (const std::string & k : keys) {628 if (valid_path && current.is_object() && current.contains(k)) {629 current = current[k];630 } else {631 valid_path = false;632 }633 }634 if (valid_path) {635 result[path] = current;636 }637 }638 return result;639}640 641llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt, bool add_special, bool parse_special) {642 // If `add_bos` is true, we only add BOS, when json_prompt is a string,643 // or the first element of the json_prompt array is a string.644 llama_tokens prompt_tokens;645 646 if (json_prompt.is_array()) {647 bool first = true;648 for (const auto & p : json_prompt) {649 if (p.is_string()) {650 auto s = p.template get<std::string>();651 652 llama_tokens p;653 if (first) {654 p = common_tokenize(vocab, s, add_special, parse_special);655 first = false;656 } else {657 p = common_tokenize(vocab, s, false, parse_special);658 }659 660 prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());661 } else {662 if (first) {663 first = false;664 }665 666 prompt_tokens.push_back(p.template get<llama_token>());667 }668 }669 } else {670 auto s = json_prompt.template get<std::string>();671 prompt_tokens = common_tokenize(vocab, s, add_special, parse_special);672 }673 674 return prompt_tokens;675}676 677size_t validate_utf8(const std::string& text) {678 size_t len = text.size();679 if (len == 0) return 0;680 681 // Check the last few bytes to see if a multi-byte character is cut off682 for (size_t i = 1; i <= 4 && i <= len; ++i) {683 unsigned char c = text[len - i];684 // Check for start of a multi-byte sequence from the end685 if ((c & 0xE0) == 0xC0) {686 // 2-byte character start: 110xxxxx687 // Needs at least 2 bytes688 if (i < 2) return len - i;689 } else if ((c & 0xF0) == 0xE0) {690 // 3-byte character start: 1110xxxx691 // Needs at least 3 bytes692 if (i < 3) return len - i;693 } else if ((c & 0xF8) == 0xF0) {694 // 4-byte character start: 11110xxx695 // Needs at least 4 bytes696 if (i < 4) return len - i;697 }698 }699 700 // If no cut-off multi-byte character is found, return full length701 return len;702}703 704// Computes FNV-1a hash of the data705static std::string fnv_hash(const uint8_t * data, size_t len) {706 const uint64_t fnv_prime = 0x100000001b3ULL;707 uint64_t hash = 0xcbf29ce484222325ULL;708 709 for (size_t i = 0; i < len; ++i) {710 hash ^= data[i];711 hash *= fnv_prime;712 }713 return std::to_string(hash);714}715 716server_tokens process_mtmd_prompt(mtmd_context * mctx, std::string prompt, std::vector<raw_buffer> files) {717 mtmd::bitmaps bitmaps;718 for (auto & file : files) {719 mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size()));720 if (!bmp.ptr) {721 throw std::runtime_error("Failed to load image or audio file");722 }723 // calculate bitmap hash (for KV caching)724 std::string hash = fnv_hash(bmp.data(), bmp.n_bytes());725 bmp.set_id(hash.c_str());726 bitmaps.entries.push_back(std::move(bmp));727 }728 // process prompt729 std::vector<server_tokens> inputs;730 // multimodal731 mtmd_input_text inp_txt = {732 prompt.c_str(),733 /* add_special */ true,734 /* parse_special */ true,735 };736 mtmd::input_chunks chunks(mtmd_input_chunks_init());737 auto bitmaps_c_ptr = bitmaps.c_ptr();738 int32_t tokenized = mtmd_tokenize(mctx,739 chunks.ptr.get(),740 &inp_txt,741 bitmaps_c_ptr.data(),742 bitmaps_c_ptr.size());743 if (tokenized != 0) {744 throw std::runtime_error("Failed to tokenize prompt");745 }746 auto result = server_tokens(chunks, true);747 return result;748}749 750/**751 * break the input "prompt" object into multiple prompt if needed, then tokenize them752 * use tokenize_input_prompts() if the input could be an array.753 * this supports these cases:754 * - "prompt": "string"755 * - "prompt": [12, 34, 56]756 * - "prompt": [12, 34, "string", 56, 78]757 * - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }758 */759static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {760 constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string";761 constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data";762 const bool has_mtmd = mctx != nullptr;763 if (json_prompt.is_string() || json_is_array_of_mixed_numbers_strings(json_prompt)) {764 // string or mixed765 llama_tokens tmp = tokenize_mixed(vocab, json_prompt, add_special, parse_special);766 return server_tokens(tmp, false);767 } else if (json_is_array_of_numbers(json_prompt)) {768 // array of tokens769 llama_tokens tmp = json_prompt.get<llama_tokens>();770 return server_tokens(tmp, false);771 } else if (json_prompt.contains(JSON_STRING_PROMPT_KEY)) {772 // JSON object with prompt key.773 if (json_prompt.contains(JSON_MTMD_DATA_KEY)) {774 if (!has_mtmd)775 throw std::runtime_error("Multimodal data provided, but model does not support multimodal requests.");776 777 // JSON object with prompt and multimodal key.778 std::vector<raw_buffer> files;779 for (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) {780 files.push_back(base64_decode(entry));781 }782 return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files);783 } else {784 // Not multimodal, but contains a subobject.785 llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special);786 return server_tokens(tmp, false);787 }788 } else {789 throw std::runtime_error("\"prompt\" elements must be a string, a list of tokens, a JSON object containing a prompt string, or a list of mixed strings & tokens.");790 }791}792 793std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) {794 std::vector<server_tokens> result;795 if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) {796 result.reserve(json_prompt.size());797 for (const auto & p : json_prompt) {798 result.push_back(tokenize_input_subprompt(vocab, mctx, p,add_special, parse_special));799 }800 } else {801 result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special));802 }803 if (result.empty()) {804 throw std::runtime_error("\"prompt\" must not be empty");805 }806 return result;807}808 809//810// OAI utils811//812 813// used by /completions endpoint814json oaicompat_completion_params_parse(const json & body) {815 json llama_params;816 817 if (!body.contains("prompt")) {818 throw std::runtime_error("\"prompt\" is required");819 }820 821 // Handle "stop" field822 if (body.contains("stop") && body.at("stop").is_string()) {823 llama_params["stop"] = json::array({body.at("stop").get<std::string>()});824 } else {825 llama_params["stop"] = json_value(body, "stop", json::array());826 }827 828 // Handle "echo" field829 if (json_value(body, "echo", false)) {830 throw std::runtime_error("Only no echo is supported");831 }832 833 // Params supported by OAI but unsupported by llama.cpp834 static const std::vector<std::string> unsupported_params { "best_of", "suffix" };835 for (const auto & param : unsupported_params) {836 if (body.contains(param)) {837 throw std::runtime_error("Unsupported param: " + param);838 }839 }840 841 // Copy remaining properties to llama_params842 for (const auto & item : body.items()) {843 // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens"844 if (!llama_params.contains(item.key()) || item.key() == "n_predict") {845 llama_params[item.key()] = item.value();846 }847 }848 849 return llama_params;850}851 852// media_path always end with '/', see arg.cpp853static void handle_media(854 std::vector<raw_buffer> & out_files,855 json & media_obj,856 const std::string & media_path) {857 std::string url = json_value(media_obj, "url", std::string());858 if (string_starts_with(url, "http")) {859 // download remote image860 // TODO @ngxson : maybe make these params configurable861 common_remote_params params;862 params.max_size = 1024 * 1024 * 10; // 10MB863 params.timeout = 10; // seconds864 SRV_INF("downloading image from '%s'\n", url.c_str());865 auto res = common_remote_get_content(url, params);866 if (200 <= res.first && res.first < 300) {867 SRV_INF("downloaded %zu bytes\n", res.second.size());868 raw_buffer data;869 data.insert(data.end(), res.second.begin(), res.second.end());870 out_files.push_back(data);871 } else {872 throw std::runtime_error("Failed to download image");873 }874 875 } else if (string_starts_with(url, "file://")) {876 if (media_path.empty()) {877 throw std::invalid_argument("file:// URLs are not allowed unless --media-path is specified");878 }879 // load local image file880 std::string file_path = url.substr(7); // remove "file://"881 raw_buffer data;882 if (!fs_validate_filename(file_path, true)) {883 throw std::invalid_argument("file path is not allowed: " + file_path);884 }885 SRV_INF("loading image from local file '%s'\n", (media_path + file_path).c_str());886 std::ifstream file(media_path + file_path, std::ios::binary);887 if (!file) {888 throw std::invalid_argument("file does not exist or cannot be opened: " + file_path);889 }890 data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());891 out_files.push_back(data);892 893 } else {894 // try to decode base64 image895 std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');896 if (parts.size() != 2) {897 throw std::runtime_error("Invalid url value");898 } else if (!string_starts_with(parts[0], "data:image/")) {899 throw std::runtime_error("Invalid url format: " + parts[0]);900 } else if (!string_ends_with(parts[0], "base64")) {901 throw std::runtime_error("url must be base64 encoded");902 } else {903 auto base64_data = parts[1];904 auto decoded_data = base64_decode(base64_data);905 out_files.push_back(decoded_data);906 }907 }908}909 910// used by /chat/completions endpoint911json oaicompat_chat_params_parse(912 json & body, /* openai api json semantics */913 const server_chat_params & opt,914 std::vector<raw_buffer> & out_files)915{916 json llama_params;917 918 auto tools = json_value(body, "tools", json());919 auto has_tools = tools.is_array() && !tools.empty();920 auto stream = json_value(body, "stream", false);921 auto tool_choice = json_value(body, "tool_choice", std::string("auto"));922 923 if (!opt.use_jinja) {924 if (has_tools) {925 throw std::runtime_error("tools param requires --jinja flag");926 }927 if (tool_choice != "auto") {928 throw std::runtime_error("tool_choice param requires --jinja flag");929 }930 }931 932 // Handle "stop" field933 if (body.contains("stop") && body.at("stop").is_string()) {934 llama_params["stop"] = json::array({body.at("stop").get<std::string>()});935 } else {936 llama_params["stop"] = json_value(body, "stop", json::array());937 }938 939 auto json_schema = json_value(body, "json_schema", json());940 auto grammar = json_value(body, "grammar", std::string());941 if (!json_schema.is_null() && !grammar.empty()) {942 throw std::runtime_error("Cannot use both json_schema and grammar");943 }944 945 // Handle "response_format" field946 if (body.contains("response_format")) {947 json response_format = json_value(body, "response_format", json::object());948 std::string response_type = json_value(response_format, "type", std::string());949 if (response_type == "json_object") {950 json_schema = json_value(response_format, "schema", json::object());951 } else if (response_type == "json_schema") {952 auto schema_wrapper = json_value(response_format, "json_schema", json::object());953 json_schema = json_value(schema_wrapper, "schema", json::object());954 } else if (!response_type.empty() && response_type != "text") {955 throw std::invalid_argument("response_format type must be one of \"text\" or \"json_object\", but got: " + response_type);956 }957 }958 959 // get input files960 if (!body.contains("messages")) {961 throw std::invalid_argument("'messages' is required");962 }963 json & messages = body.at("messages");964 if (!messages.is_array()) {965 throw std::invalid_argument("Expected 'messages' to be an array");966 }967 for (auto & msg : messages) {968 std::string role = json_value(msg, "role", std::string());969 if (role != "assistant" && !msg.contains("content")) {970 throw std::invalid_argument("All non-assistant messages must contain 'content'");971 }972 if (role == "assistant") {973 if (!msg.contains("content") && !msg.contains("tool_calls")) {974 throw std::invalid_argument("Assistant message must contain either 'content' or 'tool_calls'!");975 }976 if (!msg.contains("content")) {977 continue; // avoid errors with no content978 }979 }980 json & content = msg.at("content");981 if (content.is_string() || content.is_null()) {982 continue;983 }984 985 if (!content.is_array()) {986 throw std::invalid_argument("Expected 'content' to be a string or an array");987 }988 989 for (auto & p : content) {990 std::string type = json_value(p, "type", std::string());991 if (type == "image_url") {992 if (!opt.allow_image) {993 throw std::runtime_error("image input is not supported - hint: if this is unexpected, you may need to provide the mmproj");994 }995 996 json image_url = json_value(p, "image_url", json::object());997 handle_media(out_files, image_url, opt.media_path);998 999 p["type"] = "media_marker";1000 p["text"] = get_media_marker();1001 p.erase("image_url");1002 1003 } else if (type == "input_audio") {1004 if (!opt.allow_audio) {1005 throw std::runtime_error("audio input is not supported - hint: if this is unexpected, you may need to provide the mmproj");1006 }1007 1008 json input_audio = json_value(p, "input_audio", json::object());1009 std::string data = json_value(input_audio, "data", std::string());1010 std::string format = json_value(input_audio, "format", std::string());1011 // while we also support flac, we don't allow it here so we matches the OAI spec1012 if (format != "wav" && format != "mp3") {1013 throw std::invalid_argument("input_audio.format must be either 'wav' or 'mp3'");1014 }1015 auto decoded_data = base64_decode(data); // expected to be base64 encoded1016 out_files.push_back(decoded_data);1017 1018 // TODO: add audio_url support by reusing handle_media()1019 1020 p["type"] = "media_marker";1021 p["text"] = get_media_marker();1022 p.erase("input_audio");1023 1024 } else if (type != "text") {1025 throw std::invalid_argument("unsupported content[].type");1026 }1027 }1028 }1029 1030 common_chat_templates_inputs inputs;1031 inputs.messages = common_chat_msgs_parse_oaicompat(messages);1032 inputs.tools = common_chat_tools_parse_oaicompat(tools);1033 inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice);1034 inputs.json_schema = json_schema.is_null() ? "" : json_schema.dump();1035 inputs.grammar = grammar;1036 inputs.use_jinja = opt.use_jinja;1037 inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", false);1038 inputs.add_generation_prompt = json_value(body, "add_generation_prompt", true);1039 inputs.reasoning_format = opt.reasoning_format;1040 if (body.contains("reasoning_format")) {1041 inputs.reasoning_format = common_reasoning_format_from_name(body.at("reasoning_format").get<std::string>());1042 }1043 inputs.enable_thinking = opt.enable_thinking;1044 if (!inputs.tools.empty() && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {1045 if (body.contains("grammar")) {1046 throw std::invalid_argument("Cannot use custom grammar constraints with tools.");1047 }1048 llama_params["parse_tool_calls"] = true;1049 }1050 1051 // merge the template args provided from command line with the args provided in the user request1052 auto chat_template_kwargs_object = json_value(body, "chat_template_kwargs", json::object());1053 inputs.chat_template_kwargs = opt.chat_template_kwargs;1054 for (const auto & item : chat_template_kwargs_object.items()) {1055 inputs.chat_template_kwargs[item.key()] = item.value().dump();1056 }1057 1058 // parse the "enable_thinking" kwarg to override the default value1059 auto enable_thinking_kwarg = json_value(inputs.chat_template_kwargs, "enable_thinking", std::string(""));1060 if (enable_thinking_kwarg == "true") {1061 inputs.enable_thinking = true;1062 } else if (enable_thinking_kwarg == "false") {1063 inputs.enable_thinking = false;1064 } else if (!enable_thinking_kwarg.empty() && enable_thinking_kwarg[0] == '"') {1065 throw std::invalid_argument("invalid type for \"enable_thinking\" (expected boolean, got string)");1066 }1067 1068 // if the assistant message appears at the end of list, we do not add end-of-turn token1069 // for ex. this can be useful to modify the reasoning process in reasoning models1070 bool prefill_assistant_message = !inputs.messages.empty() && inputs.messages.back().role == "assistant" && opt.prefill_assistant;1071 common_chat_msg last_message;1072 if (prefill_assistant_message) {1073 last_message = inputs.messages.back();1074 inputs.messages.pop_back();1075 1076 /* sanity check, max one assistant message at the end of the list */1077 if (!inputs.messages.empty() && inputs.messages.back().role == "assistant"){1078 throw std::invalid_argument("Cannot have 2 or more assistant messages at the end of the list.");1079 }1080 1081 /* TODO: test this properly */1082 inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE;1083 1084 if ( inputs.enable_thinking ) {1085 throw std::invalid_argument("Assistant response prefill is incompatible with enable_thinking.");1086 }1087 1088 inputs.add_generation_prompt = true;1089 }1090 inputs.force_pure_content = opt.force_pure_content;1091 1092 // Apply chat template to the list of messages1093 auto chat_params = common_chat_templates_apply(opt.tmpls.get(), inputs);1094 1095 /* Append assistant prefilled message */1096 if (prefill_assistant_message) {1097 if (!last_message.content_parts.empty()) {1098 for (auto & p : last_message.content_parts) {1099 chat_params.prompt += p.text;1100 }1101 } else {1102 chat_params.prompt += last_message.content;1103 }1104 }1105 1106 llama_params["chat_format"] = static_cast<int>(chat_params.format);1107 llama_params["prompt"] = chat_params.prompt;1108 if (!chat_params.grammar.empty()) {1109 llama_params["grammar"] = chat_params.grammar;1110 llama_params["grammar_type"] = std::string("tool_calls");1111 }1112 llama_params["grammar_lazy"] = chat_params.grammar_lazy;1113 auto grammar_triggers = json::array();1114 for (const auto & trigger : chat_params.grammar_triggers) {1115 server_grammar_trigger ct(trigger);1116 grammar_triggers.push_back(ct.to_json());1117 }1118 llama_params["grammar_triggers"] = grammar_triggers;1119 llama_params["preserved_tokens"] = chat_params.preserved_tokens;1120 llama_params["generation_prompt"] = chat_params.generation_prompt;1121 for (const auto & stop : chat_params.additional_stops) {1122 llama_params["stop"].push_back(stop);1123 }1124 if (!chat_params.parser.empty()) {1125 llama_params["chat_parser"] = chat_params.parser;1126 }1127 1128 // Reasoning budget: pass parameters through to sampling layer1129 {1130 int reasoning_budget = opt.reasoning_budget;1131 if (reasoning_budget == -1 && body.contains("thinking_budget_tokens")) {1132 reasoning_budget = json_value(body, "thinking_budget_tokens", -1);1133 }1134 1135 if (!chat_params.thinking_end_tag.empty()) {1136 llama_params["reasoning_budget_tokens"] = reasoning_budget;1137 llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag;1138 llama_params["reasoning_budget_end_tag"] = chat_params.thinking_end_tag;1139 llama_params["reasoning_budget_message"] = opt.reasoning_budget_message;1140 }1141 }1142 1143 // Handle "logprobs" field1144 // TODO: The response format of this option is not yet OAI-compatible, but seems like no one really using it; We may need to fix it in the future1145 if (json_value(body, "logprobs", false)) {1146 if (has_tools && stream) {1147 throw std::invalid_argument("logprobs is not supported with tools + stream");1148 }1149 llama_params["n_probs"] = json_value(body, "top_logprobs", 20);1150 } else if (body.contains("top_logprobs") && !body.at("top_logprobs").is_null()) {1151 throw std::invalid_argument("top_logprobs requires logprobs to be set to true");1152 }1153 1154 // Copy remaining properties to llama_params1155 // This allows user to use llama.cpp-specific params like "mirostat", ... via OAI endpoint.1156 // See "launch_slot_with_task()" for a complete list of params supported by llama.cpp1157 for (const auto & item : body.items()) {1158 // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens"1159 if (!llama_params.contains(item.key()) || item.key() == "n_predict") {1160 llama_params[item.key()] = item.value();1161 }1162 }1163 1164 return llama_params;1165}1166 1167json convert_responses_to_chatcmpl(const json & response_body) {1168 if (!response_body.contains("input")) {1169 throw std::invalid_argument("'input' is required");1170 }1171 if (!json_value(response_body, "previous_response_id", std::string{}).empty()) {1172 throw std::invalid_argument("llama.cpp does not support 'previous_response_id'.");1173 }1174 1175 const json input_value = response_body.at("input");1176 json chatcmpl_body = response_body;1177 chatcmpl_body.erase("input");1178 std::vector<json> chatcmpl_messages;1179 1180 if (response_body.contains("instructions")) {1181 chatcmpl_messages.push_back({1182 {"role", "system"},1183 {"content", json_value(response_body, "instructions", std::string())},1184 });1185 chatcmpl_body.erase("instructions");1186 }1187 1188 if (input_value.is_string()) {1189 // #responses_create-input-text_input1190 chatcmpl_messages.push_back({1191 {"role", "user"},1192 {"content", input_value},1193 });1194 } else if (input_value.is_array()) {1195 // #responses_create-input-input_item_list1196 1197 static auto exists_and_is_array = [](const json & j, const char * key) -> bool {1198 return j.contains(key) && j.at(key).is_array();1199 };1200 static auto exists_and_is_string = [](const json & j, const char * key) -> bool {