Felipe97/llama-cpp-compiled
01.1k
1#include "common.h"2#include "download.h"3#include "log.h"4#include "llama.h"5#include "mtmd.h"6#include "mtmd-helper.h"7#include "chat.h"8#include "base64.hpp"9 10#include "server-common.h"11 12#include <random>13#include <sstream>14#include <fstream>15#include <limits>16#include <cstring>17#include <type_traits>18#include <chrono>19#include <thread>20 21#ifdef _WIN3222// windows.h defines min and max as macros, which breaks std::min and std::max23#define WIN32_LEAN_AND_MEAN24#ifndef NOMINMAX25# define NOMINMAX26#endif27#include <windows.h>28#include <io.h>29#else30#include <errno.h>31#include <fcntl.h>32#include <poll.h>33#include <unistd.h>34#endif35 36json format_error_response(const std::string & message, const enum error_type type) {37 std::string type_str;38 int code = 500;39 switch (type) {40 case ERROR_TYPE_INVALID_REQUEST:41 type_str = "invalid_request_error";42 code = 400;43 break;44 case ERROR_TYPE_AUTHENTICATION:45 type_str = "authentication_error";46 code = 401;47 break;48 case ERROR_TYPE_NOT_FOUND:49 type_str = "not_found_error";50 code = 404;51 break;52 case ERROR_TYPE_SERVER:53 type_str = "server_error";54 code = 500;55 break;56 case ERROR_TYPE_PERMISSION:57 type_str = "permission_error";58 code = 403;59 break;60 case ERROR_TYPE_NOT_SUPPORTED:61 type_str = "not_supported_error";62 code = 501;63 break;64 case ERROR_TYPE_UNAVAILABLE:65 type_str = "unavailable_error";66 code = 503;67 break;68 case ERROR_TYPE_EXCEED_CONTEXT_SIZE:69 type_str = "exceed_context_size_error";70 code = 400;71 break;72 }73 return json {74 {"code", code},75 {"message", message},76 {"type", type_str},77 };78}79 80//81// server_slot_stats82//83 84json server_slot_stats::to_json() const {85 json base = {86 {"cache_n", n_prompt_cached},87 88 {"prompt_n", n_prompt_processed},89 {"prompt_ms", t_prompt_ms()},90 {"prompt_per_token_ms", t_prompt_per_token_ms()},91 {"prompt_per_second", n_prompt_tps()},92 93 {"predicted_n", n_gen},94 {"predicted_ms", t_gen_ms()},95 {"predicted_per_token_ms", t_gen_per_token_ms()},96 {"predicted_per_second", n_gen_tps()},97 };98 99 if (n_draft_tokens > 0) {100 base["draft_n"] = n_draft_tokens;101 base["draft_n_accepted"] = n_draft_accepted;102 }103 104 return base;105}106 107//108// random string / id109//110 111std::string random_string() {112 static const std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");113 114 std::random_device rd;115 std::mt19937 generator(rd());116 117 std::string result(32, ' ');118 119 for (int i = 0; i < 32; ++i) {120 result[i] = str[generator() % str.size()];121 }122 123 return result;124}125 126std::string gen_chatcmplid() {127 return "chatcmpl-" + random_string();128}129 130std::string gen_tool_call_id() {131 return random_string();132}133 134const char * get_media_marker() {135 static const std::string marker = []() {136 // allow user to pin a reproducible marker via env var137 const char * env = getenv("LLAMA_MEDIA_MARKER");138 if (env && env[0] != '\0') {139 return std::string(env);140 }141 return std::string("<__media_") + random_string() + "__>";142 }();143 return marker.c_str();144}145 146//147// lora utils148//149 150bool lora_all_alora(const std::vector<common_adapter_lora_info> & loras) {151 bool found_alora = false;152 for (const auto & lora : loras) {153 if (lora.scale != 0) {154 if (llama_adapter_get_alora_n_invocation_tokens(lora.ptr) == 0) {155 return false;156 }157 found_alora = true;158 }159 }160 return found_alora;161}162 163bool lora_should_clear_cache(164 const std::vector<common_adapter_lora_info> & current,165 const std::vector<common_adapter_lora_info> & next) {166 167 // This should always be called after determining that the two sets are168 // _not_ equal. This assert is therefore some slightly wasted work and169 // should be safe to remove as long as this method is called correctly.170 GGML_ASSERT(!are_lora_equal(current, next));171 172 return (173 !(lora_get_enabled_ids(current).empty() || lora_all_alora(current)) ||174 !lora_all_alora(next));175}176 177std::map<int, float> parse_lora_request(const json & data) {178 std::map<int, float> lora;179 180 // set value181 for (const auto & entry : data) {182 int id = json_value(entry, "id", -1);183 float scale = json_value(entry, "scale", 0.0f);184 lora[id] = scale;185 }186 187 return lora;188}189 190bool are_lora_equal(191 const std::vector<common_adapter_lora_info> & l1,192 const std::vector<common_adapter_lora_info> & l2) {193 if (l1.size() != l2.size()) {194 return false;195 }196 for (size_t i = 0; i < l1.size(); ++i) {197 // we don't check lora.path to reduce the time complexity198 if (l1[i].scale != l2[i].scale || l1[i].ptr != l2[i].ptr) {199 return false;200 }201 }202 return true;203}204 205std::vector<size_t> lora_get_enabled_ids(const std::vector<common_adapter_lora_info> & loras) {206 std::vector<size_t> enabled_ids;207 for (size_t i = 0; i < loras.size(); ++i) {208 if (loras[i].scale > 0) {209 enabled_ids.push_back(i);210 }211 }212 return enabled_ids;213}214 215//216// base64 utils (TODO: use the base64::decode from base64.hpp)217//218 219static const std::string base64_chars =220 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"221 "abcdefghijklmnopqrstuvwxyz"222 "0123456789+/";223 224static inline bool is_base64(uint8_t c) {225 return (isalnum(c) || (c == '+') || (c == '/'));226}227 228static inline raw_buffer base64_decode(const std::string & encoded_string) {229 int i = 0;230 int j = 0;231 int in_ = 0;232 233 int in_len = encoded_string.size();234 235 uint8_t char_array_4[4];236 uint8_t char_array_3[3];237 238 raw_buffer ret;239 240 while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_])) {241 char_array_4[i++] = encoded_string[in_]; in_++;242 if (i == 4) {243 for (i = 0; i < 4; i++) {244 char_array_4[i] = base64_chars.find(char_array_4[i]);245 }246 247 char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);248 char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);249 char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];250 251 for (i = 0; (i < 3); i++) {252 ret.push_back(char_array_3[i]);253 }254 255 i = 0;256 }257 }258 259 if (i) {260 for (j = i; j < 4; j++) {261 char_array_4[j] = 0;262 }263 264 for (j = 0; j < 4; j++) {265 char_array_4[j] = base64_chars.find(char_array_4[j]);266 }267 268 char_array_3[0] = ((char_array_4[0] ) << 2) + ((char_array_4[1] & 0x30) >> 4);269 char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);270 char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];271 272 for (j = 0; j < i - 1; j++) {273 ret.push_back(char_array_3[j]);274 }275 }276 277 return ret;278}279 280//281// server_tokens implementation282//283 284namespace {285 286constexpr uint32_t SERVER_TOKENS_STATE_VERSION = 1;287 288uint32_t server_tokens_state_u32(size_t value) {289 if (value > std::numeric_limits<uint32_t>::max()) {290 throw std::runtime_error("Server tokens state is too large");291 }292 return value;293}294 295class server_tokens_state_writer {296public:297 template <typename T>298 void write(T value) {299 static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");300 const auto * ptr = reinterpret_cast<const char *>(&value);301 data.insert(data.end(), ptr, ptr + sizeof(value));302 }303 304 template <typename T>305 void write(const std::vector<T> & values) {306 static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");307 write(server_tokens_state_u32(values.size()));308 if (values.empty()) {309 return;310 }311 const auto * ptr = reinterpret_cast<const char *>(values.data());312 data.insert(data.end(), ptr, ptr + values.size() * sizeof(T));313 }314 315 void write_media_chunk(const mtmd_input_chunk * chunk) {316 size_t chunk_size = 0;317 if (mtmd_input_chunk_save(chunk, nullptr, 0, &chunk_size) != 0 || chunk_size == 0) {318 throw std::runtime_error("Cannot serialize media chunk in server tokens");319 }320 std::vector<char> chunk_data(server_tokens_state_u32(chunk_size));321 if (mtmd_input_chunk_save(chunk, chunk_data.data(), chunk_data.size(), nullptr) != 0) {322 throw std::runtime_error("Cannot serialize media chunk in server tokens");323 }324 write(chunk_data);325 }326 327 std::vector<char> take() {328 data.resize((data.size() + sizeof(llama_token) - 1) / sizeof(llama_token) * sizeof(llama_token), 0);329 return std::move(data);330 }331 332private:333 std::vector<char> data;334};335 336class server_tokens_state_reader {337public:338 server_tokens_state_reader(const char * data, size_t size) : data(data), size(size) {}339 340 template <typename T>341 T read() {342 static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");343 if (size - pos < sizeof(T)) {344 throw std::runtime_error("Unexpected end of server tokens state");345 }346 T value;347 std::memcpy(&value, data + pos, sizeof(value));348 pos += sizeof(value);349 return value;350 }351 352 template <typename T>353 std::vector<T> read_vector() {354 static_assert(std::is_trivially_copyable<T>::value, "T must be trivially copyable");355 const uint32_t n_values = read<uint32_t>();356 // reject before resizing, so that a small corrupted payload cannot request a huge allocation357 if (n_values > remaining() / sizeof(T)) {358 throw std::runtime_error("Unexpected end of server tokens state");359 }360 std::vector<T> values(n_values);361 if (n_values > 0) {362 std::memcpy(values.data(), data + pos, values.size() * sizeof(T));363 pos += values.size() * sizeof(T);364 }365 return values;366 }367 368 size_t remaining() const {369 return size - pos;370 }371 372private:373 const char * data;374 size_t size;375 size_t pos = 0;376};377 378} // namespace379 380server_tokens::server_tokens(mtmd::input_chunks & mtmd_chunks, bool has_mtmd) : has_mtmd(has_mtmd) {381 for (size_t i = 0; i < mtmd_chunks.size(); ++i) {382 push_back(mtmd_chunks[i]);383 }384}385 386server_tokens::server_tokens(const llama_tokens & tokens, bool has_mtmd) : has_mtmd(has_mtmd), tokens(tokens) {387}388 389llama_pos server_tokens::pos_next(int64_t n_tokens) const {390 if (!has_mtmd) {391 if (n_tokens < 0) {392 return tokens.size();393 }394 395 return n_tokens;396 }397 398 if (n_tokens < 0) {399 llama_pos res = tokens.size();400 401 for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {402 const auto & chunk = it->second;403 res += mtmd_input_chunk_get_n_pos(chunk.get()) - mtmd_input_chunk_get_n_tokens(chunk.get());404 }405 406 return res;407 }408 409 int64_t idx = 0;410 llama_pos pos = 0;411 412 GGML_ASSERT(n_tokens <= (int64_t)tokens.size());413 414 while (idx < n_tokens) {415 const auto media_it = map_idx_to_media.find(idx);416 if (media_it != map_idx_to_media.end()) {417 const auto & chunk = media_it->second;418 const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());419 const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());420 421 pos += n_pos;422 idx += n_tok;423 } else {424 pos++;425 idx++;426 }427 }428 429 return pos;430}431 432size_t server_tokens::size_up_to_pos(llama_pos max_pos) const {433 if (!has_mtmd) {434 return std::min((size_t)max_pos, tokens.size());435 }436 437 size_t idx = 0;438 llama_pos pos = 0;439 440 while (idx < tokens.size()) {441 const auto media_it = map_idx_to_media.find(idx);442 if (media_it != map_idx_to_media.end()) {443 const auto & chunk = media_it->second;444 const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());445 const size_t n_tok = mtmd_input_chunk_get_n_tokens(chunk.get());446 447 pos += n_pos;448 idx += n_tok;449 } else {450 pos++;451 idx++;452 }453 454 if (pos >= max_pos) {455 break;456 }457 }458 459 return idx;460}461 462std::string server_tokens::str() const {463 std::ostringstream oss;464 oss << "tokens: ";465 for (size_t idx = 0; idx < tokens.size(); ++idx) {466 llama_token t = tokens[idx];467 oss << "idx:" << idx << " ";468 if (t == LLAMA_TOKEN_NULL) {469 oss << "<embd> ";470 } else {471 oss << t << " ";472 }473 }474 oss << "\n";475 oss << "image idx: ";476 for (const auto & it : map_idx_to_media) {477 oss << it.first << ", ";478 }479 return oss.str();480}481 482const mtmd::input_chunk_ptr & server_tokens::find_chunk(size_t idx) const {483 auto it = map_idx_to_media.find(idx);484 if (it != map_idx_to_media.end()) {485 return it->second;486 }487 throw std::runtime_error("Chunk not found");488}489 490std::pair<const mtmd::input_chunk_ptr *, size_t> server_tokens::find_next_media_chunk(size_t idx) const {491 auto it = map_idx_to_media.upper_bound(idx);492 if (it != map_idx_to_media.end()) {493 return { &it->second, it->first };494 }495 return { nullptr, 0 };496}497 498void server_tokens::push_back(llama_token tok) {499 if (tok == LLAMA_TOKEN_NULL) {500 throw std::runtime_error("Invalid token");501 }502 tokens.emplace_back(tok);503}504 505void server_tokens::push_back(const mtmd_input_chunk * chunk) {506 auto type = mtmd_input_chunk_get_type(chunk);507 if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {508 GGML_ASSERT(has_mtmd);509 const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);510 size_t start_idx = tokens.size();511 for (size_t i = 0; i < n_tokens; ++i) {512 tokens.emplace_back(LLAMA_TOKEN_NULL);513 }514 mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_copy(chunk));515 map_idx_to_media[start_idx] = std::move(new_chunk);516 } else if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) {517 size_t n_tokens;518 const auto * text_tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);519 for (size_t i = 0; i < n_tokens; ++i) {520 push_back(text_tokens[i]);521 }522 } else {523 GGML_ABORT("Invalid chunk type");524 }525}526 527void server_tokens::push_back_placeholder(const mtmd_input_chunk * chunk) {528 auto type = mtmd_input_chunk_get_type(chunk);529 if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {530 GGML_ASSERT(has_mtmd);531 mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_get_placeholder(chunk));532 GGML_ASSERT(new_chunk != nullptr && "failed to create placeholder chunk");533 const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);534 size_t start_idx = tokens.size();535 for (size_t i = 0; i < n_tokens; ++i) {536 tokens.emplace_back(LLAMA_TOKEN_NULL);537 }538 map_idx_to_media[start_idx] = std::move(new_chunk);539 } else {540 push_back(chunk);541 }542}543 544void server_tokens::push_back(server_tokens & tokens) {545 size_t start_idx = size();546 for (size_t i = 0; i < tokens.size(); i++) {547 push_back(tokens[i]);548 }549 if (tokens.has_mtmd) {550 // Assert if we are copying MTMD chunks to a server_tokens that does not have mtmd.551 // We could also just check, but this will prevent silently dropping MTMD data.552 GGML_ASSERT(has_mtmd);553 for (auto it = tokens.map_idx_to_media.begin(); it != tokens.map_idx_to_media.end(); ) {554 auto * chunk = tokens.map_idx_to_media[it->first].get();555 mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_copy(chunk));556 map_idx_to_media[start_idx + it->first] = std::move(new_chunk);557 }558 }559}560 561void server_tokens::insert(const llama_tokens & inp_tokens) {562 tokens.insert(tokens.end(), inp_tokens.begin(), inp_tokens.end());563}564 565const llama_tokens & server_tokens::get_tokens() const {566 GGML_ASSERT(!has_mtmd);567 return tokens;568}569 570std::vector<char> server_tokens::serialize() const {571 static_assert(sizeof(llama_token) == sizeof(uint32_t), "unexpected llama_token size");572 573 server_tokens_state_writer writer;574 writer.write((llama_token) LLAMA_TOKEN_NULL);575 writer.write(SERVER_TOKENS_STATE_VERSION);576 writer.write(tokens);577 578 std::vector<uint32_t> media_keys;579 media_keys.reserve(map_idx_to_media.size());580 for (const auto & item : map_idx_to_media) {581 media_keys.push_back(server_tokens_state_u32(item.first));582 }583 writer.write(media_keys);584 585 for (const auto & item : map_idx_to_media) {586 writer.write_media_chunk(item.second.get());587 }588 589 return writer.take();590}591 592server_tokens server_tokens::deserialize(const llama_tokens & packed, bool has_mtmd) {593 static_assert(sizeof(llama_token) == sizeof(uint32_t), "unexpected llama_token size");594 595 if (packed.empty() || packed[0] != LLAMA_TOKEN_NULL) {596 // plain token list, as written by older versions597 return server_tokens(packed, has_mtmd);598 }599 600 server_tokens_state_reader reader(reinterpret_cast<const char *>(packed.data()), packed.size() * sizeof(llama_token));601 reader.read<llama_token>(); // format marker602 if (reader.read<uint32_t>() != SERVER_TOKENS_STATE_VERSION) {603 throw std::runtime_error("Unsupported server tokens state version");604 }605 606 const llama_tokens tokens = reader.read_vector<llama_token>();607 608 // the media start indices, followed by the media chunks in the same order609 const std::vector<uint32_t> media_keys = reader.read_vector<uint32_t>();610 if (!media_keys.empty() && !has_mtmd) {611 throw std::runtime_error("Cannot restore media tokens without an mmproj");612 }613 614 server_tokens result(tokens, has_mtmd);615 616 for (const uint32_t key : media_keys) {617 const size_t start_idx = key;618 const std::vector<char> chunk_data = reader.read_vector<char>();619 if (chunk_data.empty()) {620 throw std::runtime_error("Cannot load media chunk from server tokens state");621 }622 623 mtmd::input_chunk_ptr chunk(mtmd_input_chunk_load(chunk_data.data(), chunk_data.size()));624 if (!chunk) {625 throw std::runtime_error("Cannot load media chunk from server tokens state");626 }627 result.map_idx_to_media[start_idx] = std::move(chunk);628 }629 630 if (reader.remaining() >= sizeof(llama_token)) {631 throw std::runtime_error("Trailing data in server tokens state");632 }633 634 return result;635}636 637llama_tokens server_tokens::get_text_tokens() const {638 llama_tokens res;639 res.reserve(tokens.size());640 for (llama_token t : tokens) {641 if (t != LLAMA_TOKEN_NULL) {642 res.push_back(t);643 }644 }645 return res;646}647 648void server_tokens::set_token(llama_pos pos, llama_token id) {649 GGML_ASSERT(!has_mtmd); // only allow this if mtmd is disabled650 tokens[pos] = id;651}652 653void server_tokens::keep_first(size_t n) {654 GGML_ASSERT(n <= tokens.size());655 if (has_mtmd) {656 if (n == tokens.size()) {657 return; // nothing to do658 }659 // we throw an error if we try to remove a token in the middle of an image660 // for ex. with input of 5 text tokens and 2 images:661 // [0] [1] [2] [3] [4] [img0] [img0] [img0] [img1] [img1]662 // n 1 2 3 4 5 6 7 8 9 10663 // allowed to resize ^ ^664 // disallowed to resize ^ ^ ^665 if (n > 0) {666 // make sure we never remove tokens in the middle of an image667 // note that the case where we keep a full image at the end is allowed:668 // tokens[n - 1] == LLAMA_TOKEN_NULL && tokens[n] != LLAMA_TOKEN_NULL669 if (tokens[n - 1] == LLAMA_TOKEN_NULL && tokens[n] == LLAMA_TOKEN_NULL) {670 find_chunk(n - 1); // will throw an error if the token is not begin-of-chunk671 }672 }673 // remove all image chunks that are not used anymore674 for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ) {675 size_t idx = it->first;676 if (idx >= n) {677 it = map_idx_to_media.erase(it);678 } else {679 ++it;680 }681 }682 }683 tokens.resize(n);684}685 686std::string server_tokens::detokenize(const llama_context * ctx, bool special) const {687 llama_tokens text_tokens;688 text_tokens.reserve(tokens.size());689 for (const auto & t : tokens) {690 if (t != LLAMA_TOKEN_NULL) {691 text_tokens.push_back(t);692 }693 }694 return common_detokenize(ctx, text_tokens, special);695}696 697size_t server_tokens::get_common_prefix(const server_tokens & b) const {698 const size_t max_idx = std::min(tokens.size(), b.tokens.size());699 700 if (!has_mtmd) {701 for (size_t i = 0; i < max_idx; ++i) {702 if (tokens[i] == b.tokens[i]) {703 continue;704 }705 706 return i;707 }708 709 return max_idx;710 }711 712 for (size_t i = 0; i < max_idx; ++i) {713 const llama_token ai = tokens[i];714 const llama_token bi = b.tokens[i];715 716 if (ai == LLAMA_TOKEN_NULL && bi == LLAMA_TOKEN_NULL) {717 const auto & a_chunk = find_chunk(i);718 const auto & b_chunk = b.find_chunk(i);719 720 GGML_ASSERT(a_chunk && b_chunk);721 722 const std::string id_ai = mtmd_input_chunk_get_id(a_chunk.get());723 const std::string id_bi = mtmd_input_chunk_get_id(b_chunk.get());724 725 const size_t n_tok_a = mtmd_input_chunk_get_n_tokens(a_chunk.get());726 const size_t n_tok_b = mtmd_input_chunk_get_n_tokens(b_chunk.get());727 728 if (id_ai == id_bi && n_tok_a == n_tok_b) {729 GGML_ASSERT(n_tok_a > 0 && "Invalid media chunk"); // should never happen730 i += n_tok_a - 1; // will be +1 by the for loop731 continue;732 }733 734 return i;735 }736 737 if (ai == bi) {738 continue;739 }740 741 return i;742 }743 744 return max_idx; // all tokens are equal745}746 747common_chat_msg_spans server_tokens::find_message_spans(const common_chat_msg_delimiters & delims) const {748 std::map<size_t, size_t> skips;749 for (const auto & it : map_idx_to_media) {750 skips[it.first] = mtmd_input_chunk_get_n_tokens(it.second.get());751 }752 return delims.split(tokens, skips);753}754 755bool server_tokens::validate(const struct llama_context * ctx) const {756 const llama_model * model = llama_get_model(ctx);757 const llama_vocab * vocab = llama_model_get_vocab(model);758 const int32_t n_vocab = llama_vocab_n_tokens(vocab);759 size_t n_media = 0;760 761 for (size_t i = 0; i < tokens.size(); ++i) {762 const auto & t = tokens[i];763 if (t == LLAMA_TOKEN_NULL) {764 try {765 const auto & chunk = find_chunk(i);766 if (mtmd_input_chunk_get_type(chunk.get()) == MTMD_INPUT_CHUNK_TYPE_TEXT) {767 return false;768 }769 const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk.get());770 const llama_pos n_pos = mtmd_input_chunk_get_n_pos(chunk.get());771 if (n_tokens == 0 || n_pos <= 0 || n_tokens > tokens.size() - i) {772 return false;773 }774 for (size_t j = i; j < i + n_tokens; ++j) {775 if (tokens[j] != LLAMA_TOKEN_NULL) {776 return false;777 }778 }779 ++n_media;780 i += n_tokens - 1;781 } catch (const std::exception & e) {782 return false;783 }784 } else if (t < 0 || t >= n_vocab) {785 return false;786 }787 }788 return n_media == map_idx_to_media.size();789}790 791server_tokens server_tokens::clone() const {792 server_tokens res;793 res.has_mtmd = has_mtmd;794 res.tokens = tokens;795 for (auto it = map_idx_to_media.begin(); it != map_idx_to_media.end(); ++it) {796 size_t idx = it->first;797 const mtmd::input_chunk_ptr & chunk = it->second;798 res.map_idx_to_media[idx] = mtmd::input_chunk_ptr(mtmd_input_chunk_copy(chunk.get()));799 }800 return res;801}802 803//804// tokenizer and input processing utils805//806 807bool json_is_array_of_numbers(const json & data) {808 if (data.is_array()) {809 for (const auto & e : data) {810 if (!e.is_number_integer()) {811 return false;812 }813 }814 return true;815 }816 return false;817}818 819bool json_is_array_of_mixed_numbers_strings(const json & data) {820 bool seen_string = false;821 bool seen_number = false;822 if (data.is_array()) {823 for (const auto & e : data) {824 seen_string |= e.is_string();825 seen_number |= e.is_number_integer();826 if (seen_number && seen_string) {827 return true;828 }829 }830 }831 return false;832}833 834bool json_is_array_and_contains_numbers(const json & data) {835 if (data.is_array()) {836 for (const auto & e : data) {837 if (e.is_number_integer()) {838 return true;839 }840 }841 return false;842 }843 return false;844}845 846json json_get_nested_values(const std::vector<std::string> & paths, const json & js) {847 json result = json::object();848 849 for (const std::string & path : paths) {850 json current = js;851 const auto keys = string_split<std::string>(path, /*separator*/ '/');852 bool valid_path = true;853 for (const std::string & k : keys) {854 if (valid_path && current.is_object() && current.contains(k)) {855 current = current[k];856 } else {857 valid_path = false;858 }859 }860 if (valid_path) {861 result[path] = current;862 }863 }864 return result;865}866 867llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt, bool add_special, bool parse_special) {868 // If `add_bos` is true, we only add BOS, when json_prompt is a string,869 // or the first element of the json_prompt array is a string.870 llama_tokens prompt_tokens;871 872 if (json_prompt.is_array()) {873 bool first = true;874 for (const auto & p : json_prompt) {875 if (p.is_string()) {876 auto s = p.template get<std::string>();877 878 llama_tokens p;879 if (first) {880 p = common_tokenize(vocab, s, add_special, parse_special);881 first = false;882 } else {883 p = common_tokenize(vocab, s, false, parse_special);884 }885 886 prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());887 } else {888 if (first) {889 first = false;890 }891 892 prompt_tokens.push_back(p.template get<llama_token>());893 }894 }895 } else {896 auto s = json_prompt.template get<std::string>();897 prompt_tokens = common_tokenize(vocab, s, add_special, parse_special);898 }899 900 return prompt_tokens;901}902 903size_t validate_utf8(const std::string& text) {904 size_t len = text.size();905 if (len == 0) return 0;906 907 // Check the last few bytes to see if a multi-byte character is cut off908 for (size_t i = 1; i <= 4 && i <= len; ++i) {909 unsigned char c = text[len - i];910 // Check for start of a multi-byte sequence from the end911 if ((c & 0xE0) == 0xC0) {912 // 2-byte character start: 110xxxxx913 // Needs at least 2 bytes914 if (i < 2) return len - i;915 } else if ((c & 0xF0) == 0xE0) {916 // 3-byte character start: 1110xxxx917 // Needs at least 3 bytes918 if (i < 3) return len - i;919 } else if ((c & 0xF8) == 0xF0) {920 // 4-byte character start: 11110xxx921 // Needs at least 4 bytes922 if (i < 4) return len - i;923 }924 }925 926 // If no cut-off multi-byte character is found, return full length927 return len;928}929 930server_tokens process_mtmd_prompt(931 mtmd_context * mctx,932 const std::string & prompt,933 const std::vector<raw_buffer> & files,934 const mtmd_helper_init_opt & init_opt,935 bool is_placeholder) {936 // these will be freed upon going out of scope937 mtmd::bitmaps bitmaps;938 std::vector<mtmd_helper::video_ptr> videos;939 for (auto & file : files) {940 auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder, init_opt);941 if (!out.bitmap) {942 throw std::runtime_error("Failed to load image or audio file");943 }944 bitmaps.entries.emplace_back(out.bitmap);945 if (out.video_ctx) {946 videos.emplace_back(out.video_ctx);947 }948 }949 // process prompt950 std::vector<server_tokens> inputs;951 // multimodal952 mtmd_input_text inp_txt = {953 prompt.data(),954 prompt.size(),955 /* add_special */ true,956 /* parse_special */ true,957 };958 mtmd::input_chunks chunks(mtmd_input_chunks_init());959 auto bitmaps_c_ptr = bitmaps.c_ptr();960 int32_t tokenized = mtmd_tokenize(mctx,961 chunks.ptr.get(),962 &inp_txt,963 bitmaps_c_ptr.data(),964 bitmaps_c_ptr.size());965 if (tokenized != 0) {966 throw std::runtime_error("Failed to tokenize prompt");967 }968 auto result = server_tokens(chunks, true);969 return result;970}971 972/**973 * break the input "prompt" object into multiple prompt if needed, then tokenize them974 * use tokenize_input_prompts() if the input could be an array.975 * this supports these cases:976 * - "prompt": "string"977 * - "prompt": [12, 34, 56]978 * - "prompt": [12, 34, "string", 56, 78]979 * - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }980 */981static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {982 constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string";983 constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data";984 const bool has_mtmd = mctx != nullptr;985 if (json_prompt.is_string() || json_is_array_of_mixed_numbers_strings(json_prompt)) {986 // string or mixed987 llama_tokens tmp = tokenize_mixed(vocab, json_prompt, add_special, parse_special);988 return server_tokens(tmp, false);989 } else if (json_is_array_of_numbers(json_prompt)) {990 // array of tokens991 llama_tokens tmp = json_prompt.get<llama_tokens>();992 return server_tokens(tmp, false);993 } else if (json_prompt.contains(JSON_STRING_PROMPT_KEY)) {994 // JSON object with prompt key.995 if (json_prompt.contains(JSON_MTMD_DATA_KEY)) {996 if (!has_mtmd)997 throw std::runtime_error("Multimodal data provided, but model does not support multimodal requests.");998 999 // JSON object with prompt and multimodal key.1000 std::vector<raw_buffer> files;1001 for (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) {1002 files.push_back(base64_decode(entry));1003 }1004 return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files, init_opt);1005 } else {1006 // Not multimodal, but contains a subobject.1007 llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special);1008 return server_tokens(tmp, false);1009 }1010 } else {1011 throw std::runtime_error("\"prompt\" elements must be a string, a list of tokens, a JSON object containing a prompt string, or a list of mixed strings & tokens.");1012 }1013}1014 1015std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {1016 std::vector<server_tokens> result;1017 if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) {1018 result.reserve(json_prompt.size());1019 for (const auto & p : json_prompt) {1020 result.push_back(tokenize_input_subprompt(vocab, mctx, p, add_special, parse_special, init_opt));1021 }1022 } else {1023 result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special, init_opt));1024 }1025 if (result.empty()) {1026 throw std::runtime_error("\"prompt\" must not be empty");1027 }1028 return result;1029}1030 1031//1032// OAI utils1033//1034 1035// used by /completions endpoint1036json oaicompat_completion_params_parse(const json & body) {1037 json llama_params;1038 1039 if (!body.contains("prompt")) {1040 throw std::runtime_error("\"prompt\" is required");1041 }1042 1043 // Handle "stop" field1044 if (body.contains("stop") && body.at("stop").is_string()) {1045 llama_params["stop"] = json::array({body.at("stop").get<std::string>()});1046 } else {1047 llama_params["stop"] = json_value(body, "stop", json::array());1048 }1049 1050 // Handle "echo" field1051 if (json_value(body, "echo", false)) {1052 throw std::runtime_error("Only no echo is supported");1053 }1054 1055 // Params supported by OAI but unsupported by llama.cpp1056 static const std::vector<std::string> unsupported_params { "best_of", "suffix" };1057 for (const auto & param : unsupported_params) {1058 if (body.contains(param)) {1059 throw std::runtime_error("Unsupported param: " + param);1060 }1061 }1062 1063 // Copy remaining properties to llama_params1064 for (const auto & item : body.items()) {1065 // Exception: if "n_predict" is present, we overwrite the value specified earlier by "max_tokens"1066 if (!llama_params.contains(item.key()) || item.key() == "n_predict") {1067 llama_params[item.key()] = item.value();1068 }1069 }1070 1071 return llama_params;1072}1073 1074// url can be1075// - http(s):// for remote files1076// - file:// for local files (only allowed if media_path is set)1077// - data: for base64 encoded data with uri scheme (e.g. data:image/png;base64,...)1078// - raw base64 encoded data1079static void handle_media(1080 std::vector<raw_buffer> & out_files,1081 const std::string & url,1082 const std::string & media_path) {1083 if (!media_path.empty()) {1084 // should already be enforced by arg.cpp, but checking just in case1085 GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR);1086 }1087 1088 if (string_starts_with(url, "http")) {1089 // download remote image1090 // TODO @ngxson : maybe make these params configurable1091 common_remote_params params;1092 params.max_size = 1024 * 1024 * 10; // 10MB1093 params.timeout = 10; // seconds1094 SRV_INF("downloading image from '%s'\n", url.c_str());1095 auto res = common_remote_get_content(url, params);1096 if (200 <= res.first && res.first < 300) {1097 SRV_INF("downloaded %zu bytes\n", res.second.size());1098 raw_buffer data;1099 data.insert(data.end(), res.second.begin(), res.second.end());1100 out_files.push_back(data);1101 } else {1102 throw std::runtime_error("Failed to download image");1103 }1104 1105 } else if (string_starts_with(url, "file://")) {1106 if (media_path.empty()) {1107 throw std::invalid_argument("file:// URLs are not allowed unless --media-path is specified");1108 }1109 // load local image file1110 std::string file_path = url.substr(7); // remove "file://"1111 raw_buffer data;1112 if (!fs_validate_filename(file_path, true)) {1113 throw std::invalid_argument("file path is not allowed: " + file_path);1114 }1115 SRV_INF("loading image from local file '%s'\n", (media_path + file_path).c_str());1116 std::ifstream file(media_path + file_path, std::ios::binary);1117 if (!file) {1118 throw std::invalid_argument("file does not exist or cannot be opened: " + file_path);1119 }1120 data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());1121 out_files.push_back(data);1122 1123 } else if (string_starts_with(url, "data:")) {1124 // try to decode base64 image, video, or audio1125 std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');1126 if (parts.size() != 2) {1127 throw std::invalid_argument("Invalid uri-encoded base64 value");1128 } else if (!string_starts_with(parts[0], "data:image/")1129 && !string_starts_with(parts[0], "data:video/")1130 && !string_starts_with(parts[0], "data:audio/")) {1131 throw std::invalid_argument("Invalid uri format: " + parts[0]);1132 } else if (!string_ends_with(parts[0], "base64")) {1133 throw std::invalid_argument("uri must be base64 encoded");1134 } else {1135 auto base64_data = parts[1];1136 auto decoded_data = base64_decode(base64_data);1137 out_files.push_back(decoded_data);1138 }1139 1140 } else {1141 // try as raw base64 string1142 auto decoded_data = base64_decode(url);1143 if (decoded_data.empty()) {1144 throw std::runtime_error("Invalid base64 value");1145 }1146 out_files.push_back(decoded_data);1147 }1148}1149 1150// used by /chat/completions endpoint1151json oaicompat_chat_params_parse(1152 json & body, /* openai api json semantics */1153 const server_chat_params & opt,1154 std::vector<raw_buffer> & out_files)1155{1156 json llama_params;1157 1158 auto tools = json_value(body, "tools", json());1159 auto has_tools = tools.is_array() && !tools.empty();1160 auto stream = json_value(body, "stream", false);1161 auto tool_choice = json_value(body, "tool_choice", std::string("auto"));1162 1163 if (!opt.use_jinja) {1164 if (has_tools) {1165 throw std::runtime_error("tools param requires --jinja flag");1166 }1167 if (tool_choice != "auto") {1168 throw std::runtime_error("tool_choice param requires --jinja flag");1169 }1170 }1171 1172 // Handle "stop" field1173 if (body.contains("stop") && body.at("stop").is_string()) {1174 llama_params["stop"] = json::array({body.at("stop").get<std::string>()});1175 } else {1176 llama_params["stop"] = json_value(body, "stop", json::array());1177 }1178 1179 auto json_schema = json_value(body, "json_schema", json());1180 auto grammar = json_value(body, "grammar", std::string());1181 if (!json_schema.is_null() && !grammar.empty()) {1182 throw std::runtime_error("Cannot use both json_schema and grammar");1183 }1184 1185 // Handle "response_format" field1186 if (body.contains("response_format")) {1187 json response_format = json_value(body, "response_format", json::object());1188 std::string response_type = json_value(response_format, "type", std::string());1189 if (response_type == "json_object") {1190 if (response_format.contains("schema") || json_schema.empty()) {1191 json_schema = json_value(response_format, "schema", json::object());1192 }1193 } else if (response_type == "json_schema") {1194 auto schema_wrapper = json_value(response_format, "json_schema", json::object());1195 json_schema = json_value(schema_wrapper, "schema", json::object());1196 } else if (!response_type.empty() && response_type != "text") {1197 throw std::invalid_argument("response_format type must be one of \"text\" or \"json_object\", but got: " + response_type);1198 }1199 }1200