Felipe97/llama-cpp-compiled
01.1k
1#include "server-chat.h"2#include "server-common.h"3 4#include <sstream>5 6json server_chat_convert_responses_to_chatcmpl(const json & response_body) {7 if (!response_body.contains("input")) {8 throw std::invalid_argument("'input' is required");9 }10 if (!json_value(response_body, "previous_response_id", std::string{}).empty()) {11 throw std::invalid_argument("llama.cpp does not support 'previous_response_id'.");12 }13 14 const json input_value = response_body.at("input");15 json chatcmpl_body = response_body;16 chatcmpl_body.erase("input");17 std::vector<json> chatcmpl_messages;18 19 if (response_body.contains("instructions")) {20 chatcmpl_messages.push_back({21 {"role", "system"},22 {"content", json_value(response_body, "instructions", std::string())},23 });24 chatcmpl_body.erase("instructions");25 }26 27 if (input_value.is_string()) {28 // #responses_create-input-text_input29 chatcmpl_messages.push_back({30 {"role", "user"},31 {"content", input_value},32 });33 } else if (input_value.is_array()) {34 // #responses_create-input-input_item_list35 36 static auto exists_and_is_array = [](const json & j, const char * key) -> bool {37 return j.contains(key) && j.at(key).is_array();38 };39 static auto exists_and_is_string = [](const json & j, const char * key) -> bool {40 return j.contains(key) && j.at(key).is_string();41 };42 43 for (json item : input_value) {44 bool merge_prev = !chatcmpl_messages.empty() && chatcmpl_messages.back().value("role", "") == "assistant";45 46 if (exists_and_is_string(item, "content")) {47 // #responses_create-input-input_item_list-input_message-content-text_input48 // Only "Input message" contains item["content"]::string49 // After converting item["content"]::string to item["content"]::array,50 // we can treat "Input message" as sum of "Item-Input message" and "Item-Output message"51 item["content"] = json::array({52 json {53 {"text", item.at("content")},54 {"type", "input_text"}55 }56 });57 }58 59 if (exists_and_is_array(item, "content") &&60 exists_and_is_string(item, "role") &&61 (item.at("role") == "user" ||62 item.at("role") == "system" ||63 item.at("role") == "developer")64 ) {65 // #responses_create-input-input_item_list-item-input_message66 std::vector<json> chatcmpl_content;67 68 for (const json & input_item : item.at("content")) {69 const std::string type = json_value(input_item, "type", std::string());70 71 if (type == "input_text") {72 if (!input_item.contains("text")) {73 throw std::invalid_argument("'Input text' requires 'text'");74 }75 chatcmpl_content.push_back({76 {"text", input_item.at("text")},77 {"type", "text"},78 });79 } else if (type == "input_image") {80 // While `detail` is marked as required,81 // it has default value("auto") and can be omitted.82 83 if (!input_item.contains("image_url")) {84 throw std::invalid_argument("'image_url' is required");85 }86 chatcmpl_content.push_back({87 {"image_url", json {88 {"url", input_item.at("image_url")}89 }},90 {"type", "image_url"},91 });92 } else if (type == "input_file") {93 throw std::invalid_argument("'input_file' is not supported by llamacpp at this moment");94 } else {95 throw std::invalid_argument("'type' must be one of 'input_text', 'input_image', or 'input_file'");96 }97 }98 99 if (item.contains("type")) {100 item.erase("type");101 }102 if (item.contains("status")) {103 item.erase("status");104 }105 item["content"] = chatcmpl_content;106 107 chatcmpl_messages.push_back(item);108 } else if (exists_and_is_string(item, "role") &&109 item.at("role") == "assistant" &&110 exists_and_is_string(item, "type") &&111 item.at("type") == "message"112 ) {113 // #responses_create-input-input_item_list-item-output_message114 auto chatcmpl_content = json::array();115 116 // Handle both string content and array content117 if (item.contains("content") && item.at("content").is_string()) {118 // String content - convert to text content part119 chatcmpl_content.push_back({120 {"text", item.at("content")},121 {"type", "text"},122 });123 } else if (exists_and_is_array(item, "content")) {124 // Array content - process each item125 for (const auto & output_text : item.at("content")) {126 const std::string type = json_value(output_text, "type", std::string());127 if (type == "output_text" || type == "input_text") {128 // Accept both output_text and input_text (string content gets converted to input_text)129 if (!exists_and_is_string(output_text, "text")) {130 throw std::invalid_argument("'Output text' requires 'text'");131 }132 chatcmpl_content.push_back({133 {"text", output_text.at("text")},134 {"type", "text"},135 });136 } else if (type == "refusal") {137 if (!exists_and_is_string(output_text, "refusal")) {138 throw std::invalid_argument("'Refusal' requires 'refusal'");139 }140 chatcmpl_content.push_back({141 {"refusal", output_text.at("refusal")},142 {"type", "refusal"},143 });144 } else {145 throw std::invalid_argument("'type' must be one of 'output_text' or 'refusal'");146 }147 }148 }149 150 if (merge_prev) {151 auto & prev_msg = chatcmpl_messages.back();152 if (!exists_and_is_array(prev_msg, "content")) {153 prev_msg["content"] = json::array();154 }155 auto & prev_content = prev_msg["content"];156 prev_content.insert(chatcmpl_content);157 } else {158 item.erase("status");159 item.erase("type");160 item["content"] = chatcmpl_content;161 chatcmpl_messages.push_back(item);162 }163 } else if (exists_and_is_string(item, "arguments") &&164 exists_and_is_string(item, "call_id") &&165 exists_and_is_string(item, "name") &&166 exists_and_is_string(item, "type") &&167 item.at("type") == "function_call"168 ) {169 // #responses_create-input-input_item_list-item-function_tool_call170 json tool_call = {171 {"function", json {172 {"arguments", item.at("arguments")},173 {"name", item.at("name")},174 }},175 {"id", item.at("call_id")},176 {"type", "function"},177 };178 179 if (merge_prev) {180 auto & prev_msg = chatcmpl_messages.back();181 if (!exists_and_is_array(prev_msg, "tool_calls")) {182 prev_msg["tool_calls"] = json::array();183 }184 prev_msg["tool_calls"].push_back(tool_call);185 } else {186 chatcmpl_messages.push_back(json {187 {"role", "assistant"},188 {"tool_calls", json::array({tool_call})}189 });190 }191 } else if (exists_and_is_string(item, "call_id") &&192 (exists_and_is_string(item, "output") || exists_and_is_array(item, "output")) &&193 exists_and_is_string(item, "type") &&194 item.at("type") == "function_call_output"195 ) {196 // #responses_create-input-input_item_list-item-function_tool_call_output197 if (item.at("output").is_string()) {198 chatcmpl_messages.push_back(json {199 {"content", item.at("output")},200 {"role", "tool"},201 {"tool_call_id", item.at("call_id")},202 });203 } else {204 json chatcmpl_outputs = item.at("output");205 for (json & chatcmpl_output : chatcmpl_outputs) {206 if (!chatcmpl_output.contains("type") || chatcmpl_output.at("type") != "input_text") {207 throw std::invalid_argument("Output of tool call should be 'Input text'");208 }209 chatcmpl_output["type"] = "text";210 }211 chatcmpl_messages.push_back(json {212 {"content", chatcmpl_outputs},213 {"role", "tool"},214 {"tool_call_id", item.at("call_id")},215 });216 }217 } else if (exists_and_is_array(item, "summary") &&218 exists_and_is_string(item, "type") &&219 item.at("type") == "reasoning") {220 // #responses_create-input-input_item_list-item-reasoning221 222 if (!exists_and_is_array(item, "content")) {223 throw std::invalid_argument("item['content'] is not an array");224 }225 if (item.at("content").empty()) {226 throw std::invalid_argument("item['content'] is empty");227 }228 if (!exists_and_is_string(item.at("content")[0], "text")) {229 throw std::invalid_argument("item['content']['text'] is not a string");230 }231 232 if (merge_prev) {233 auto & prev_msg = chatcmpl_messages.back();234 prev_msg["reasoning_content"] = item.at("content")[0].at("text");235 } else {236 chatcmpl_messages.push_back(json {237 {"role", "assistant"},238 {"content", json::array()},239 {"reasoning_content", item.at("content")[0].at("text")},240 });241 }242 } else {243 throw std::invalid_argument("Cannot determine type of 'item'");244 }245 }246 } else {247 throw std::invalid_argument("'input' must be a string or array of objects");248 }249 250 chatcmpl_body["messages"] = chatcmpl_messages;251 252 if (response_body.contains("tools")) {253 if (!response_body.at("tools").is_array()) {254 throw std::invalid_argument("'tools' must be an array of objects");255 }256 std::vector<json> chatcmpl_tools;257 for (json resp_tool : response_body.at("tools")) {258 json chatcmpl_tool;259 260 const std::string type = json_value(resp_tool, "type", std::string());261 if (type != "function") {262 // Non-function Responses tools have no Chat Completions equivalent.263 SRV_WRN("unsupported Responses tool type '%s' skipped\n", type.c_str());264 continue;265 }266 resp_tool.erase("type");267 chatcmpl_tool["type"] = "function";268 269 if (!resp_tool.contains("strict")) {270 resp_tool["strict"] = true;271 }272 chatcmpl_tool["function"] = resp_tool;273 chatcmpl_tools.push_back(chatcmpl_tool);274 }275 chatcmpl_body.erase("tools");276 if (!chatcmpl_tools.empty()) {277 chatcmpl_body["tools"] = chatcmpl_tools;278 }279 }280 281 if (response_body.contains("max_output_tokens")) {282 chatcmpl_body.erase("max_output_tokens");283 chatcmpl_body["max_tokens"] = response_body["max_output_tokens"];284 }285 286 if (response_body.contains("reasoning")) {287 // Only "effort" is handled so far288 const json & reasoning = response_body.at("reasoning");289 if (reasoning.contains("effort")) {290 chatcmpl_body["reasoning_effort"] = reasoning.at("effort");291 }292 chatcmpl_body.erase("reasoning");293 }294 295 return chatcmpl_body;296}297 298// Edits the cch section of an "x-anthropic-billing-header" system prompt.299// Does nothing to any other prompt.300//301// This is a claude message with a "cch=ef01a" attribute that breaks prefix caching.302// The cch stamp is a whitebox end-to-end integrity hint. It's not meaningful as a303// system prompt data, particularly to llama.cpp, but its presence means the prefix304// cache will not get past it: It changes on each request.305//306// Reference: https://github.com/ggml-org/llama.cpp/pull/21793307// Example header:308// ```309// x-anthropic-billing-header: cc_version=2.1.101.e51; cc_entrypoint=cli; cch=a5145;You are Claude Code, Anthropic's official CLI for Claude.310// ^^^^^311// ```312static void normalize_anthropic_billing_header(std::string & system_text) {313 if (system_text.rfind("x-anthropic-billing-header:", 0) != 0) {314 return;315 }316 317 const size_t header_prefix_length = strlen("x-anthropic-billing-header:");318 const size_t cch_length = 5;319 const size_t index_cch = system_text.find("cch=", header_prefix_length);320 if (index_cch == std::string::npos) {321 return;322 }323 324 const size_t index_replace = index_cch + 4;325 if (index_replace + cch_length < system_text.length() && system_text[index_replace + cch_length] == ';') {326 for (size_t i = 0; i < cch_length; ++i) {327 system_text[index_replace + i] = 'f';328 }329 } else {330 LOG_ERR("anthropic string not as expected: %s", system_text.c_str());331 }332}333 334json server_chat_convert_anthropic_to_oai(const json & body) {335 json oai_body;336 337 // Convert system prompt338 json oai_messages = json::array();339 auto system_param = json_value(body, "system", json());340 if (!system_param.is_null()) {341 std::string system_content;342 343 if (system_param.is_string()) {344 system_content = system_param.get<std::string>();345 normalize_anthropic_billing_header(system_content);346 } else if (system_param.is_array()) {347 for (const auto & block : system_param) {348 if (json_value(block, "type", std::string()) == "text") {349 auto system_text = json_value(block, "text", std::string());350 normalize_anthropic_billing_header(system_text);351 system_content += system_text;352 }353 }354 }355 356 oai_messages.push_back({357 {"role", "system"},358 {"content", system_content}359 });360 }361 362 // Convert messages363 if (!body.contains("messages")) {364 throw std::runtime_error("'messages' is required");365 }366 const json & messages = body.at("messages");367 if (messages.is_array()) {368 for (const auto & msg : messages) {369 std::string role = json_value(msg, "role", std::string());370 371 if (!msg.contains("content")) {372 if (role == "assistant") {373 continue;374 }375 oai_messages.push_back(msg);376 continue;377 }378 379 const json & content = msg.at("content");380 381 if (content.is_string()) {382 oai_messages.push_back(msg);383 continue;384 }385 386 if (!content.is_array()) {387 oai_messages.push_back(msg);388 continue;389 }390 391 json tool_calls = json::array();392 json converted_content = json::array();393 json tool_results = json::array();394 std::string reasoning_content;395 bool has_tool_calls = false;396 397 for (const auto & block : content) {398 std::string type = json_value(block, "type", std::string());399 400 if (type == "text") {401 converted_content.push_back(block);402 } else if (type == "thinking") {403 reasoning_content += json_value(block, "thinking", std::string());404 } else if (type == "image") {405 json source = json_value(block, "source", json::object());406 std::string source_type = json_value(source, "type", std::string());407 408 if (source_type == "base64") {409 std::string media_type = json_value(source, "media_type", std::string("image/jpeg"));410 std::string data = json_value(source, "data", std::string());411 std::ostringstream ss;412 ss << "data:" << media_type << ";base64," << data;413 414 converted_content.push_back({415 {"type", "image_url"},416 {"image_url", {417 {"url", ss.str()}418 }}419 });420 } else if (source_type == "url") {421 std::string url = json_value(source, "url", std::string());422 converted_content.push_back({423 {"type", "image_url"},424 {"image_url", {425 {"url", url}426 }}427 });428 }429 } else if (type == "tool_use") {430 tool_calls.push_back({431 {"id", json_value(block, "id", std::string())},432 {"type", "function"},433 {"function", {434 {"name", json_value(block, "name", std::string())},435 {"arguments", json_value(block, "input", json::object()).dump()}436 }}437 });438 has_tool_calls = true;439 } else if (type == "tool_result") {440 std::string tool_use_id = json_value(block, "tool_use_id", std::string());441 442 auto result_content = json_value(block, "content", json());443 if (result_content.is_string()) {444 tool_results.push_back({445 {"role", "tool"},446 {"tool_call_id", tool_use_id},447 {"content", result_content.get<std::string>()}448 });449 } else if (result_content.is_array()) {450 // Single-pass: build both text and content_parts, decide format at the end451 std::string result_text;452 json content_parts = json::array();453 bool has_images = false;454 455 for (const auto & c : result_content) {456 std::string c_type = json_value(c, "type", std::string());457 if (c_type == "text") {458 std::string text = json_value(c, "text", std::string());459 result_text += text;460 content_parts.push_back({461 {"type", "text"},462 {"text", text}463 });464 } else if (c_type == "image") {465 has_images = true;466 json source = json_value(c, "source", json::object());467 std::string source_type = json_value(source, "type", std::string());468 if (source_type == "base64") {469 std::string media_type = json_value(source, "media_type", std::string("image/jpeg"));470 std::string data = json_value(source, "data", std::string());471 std::string url = "data:" + media_type + ";base64," + data;472 content_parts.push_back({473 {"type", "image_url"},474 {"image_url", {{"url", url}}}475 });476 } else if (source_type == "url") {477 content_parts.push_back({478 {"type", "image_url"},479 {"image_url", {{"url", json_value(source, "url", std::string())}}}480 });481 }482 }483 }484 485 if (!has_images) {486 // Text-only: collapse to a plain string for maximum compatibility487 tool_results.push_back({488 {"role", "tool"},489 {"tool_call_id", tool_use_id},490 {"content", result_text}491 });492 } else {493 // Mixed or image-only: use array content parts (OpenAI multimodal tool format)494 tool_results.push_back({495 {"role", "tool"},496 {"tool_call_id", tool_use_id},497 {"content", content_parts}498 });499 }500 } else {501 tool_results.push_back({502 {"role", "tool"},503 {"tool_call_id", tool_use_id},504 {"content", ""}505 });506 }507 }508 }509 510 if (!converted_content.empty() || has_tool_calls || !reasoning_content.empty()) {511 json new_msg = {{"role", role}};512 if (!converted_content.empty()) {513 new_msg["content"] = converted_content;514 } else if (has_tool_calls || !reasoning_content.empty()) {515 new_msg["content"] = "";516 }517 if (!tool_calls.empty()) {518 new_msg["tool_calls"] = tool_calls;519 }520 if (!reasoning_content.empty()) {521 new_msg["reasoning_content"] = reasoning_content;522 }523 oai_messages.push_back(new_msg);524 }525 526 for (const auto & tool_msg : tool_results) {527 oai_messages.push_back(tool_msg);528 }529 }530 }531 532 oai_body["messages"] = oai_messages;533 534 // Convert tools535 if (body.contains("tools")) {536 const json & tools = body.at("tools");537 if (tools.is_array()) {538 json oai_tools = json::array();539 for (const auto & tool : tools) {540 oai_tools.push_back({541 {"type", "function"},542 {"function", {543 {"name", json_value(tool, "name", std::string())},544 {"description", json_value(tool, "description", std::string())},545 {"parameters", tool.contains("input_schema") ? tool.at("input_schema") : json::object()}546 }}547 });548 }549 oai_body["tools"] = oai_tools;550 }551 }552 553 // Convert tool_choice554 if (body.contains("tool_choice")) {555 const json & tc = body.at("tool_choice");556 if (tc.is_object()) {557 std::string type = json_value(tc, "type", std::string());558 if (type == "auto") {559 oai_body["tool_choice"] = "auto";560 } else if (type == "any" || type == "tool") {561 oai_body["tool_choice"] = "required";562 }563 }564 }565 566 // Convert stop_sequences to stop567 if (body.contains("stop_sequences")) {568 oai_body["stop"] = body.at("stop_sequences");569 }570 571 // Handle max_tokens (required in Anthropic, but we're permissive)572 if (body.contains("max_tokens")) {573 oai_body["max_tokens"] = body.at("max_tokens");574 } else {575 oai_body["max_tokens"] = 4096;576 }577 578 // Pass through common params579 for (const auto & key : {"temperature", "top_p", "top_k", "stream", "chat_template_kwargs"}) {580 if (body.contains(key)) {581 oai_body[key] = body.at(key);582 }583 }584 585 // Handle Anthropic-specific thinking param586 if (body.contains("thinking")) {587 json thinking = json_value(body, "thinking", json::object());588 std::string thinking_type = json_value(thinking, "type", std::string());589 if (thinking_type == "enabled") {590 int budget_tokens = json_value(thinking, "budget_tokens", 10000);591 oai_body["thinking_budget_tokens"] = budget_tokens;592 }593 }594 595 // Handle Anthropic-specific metadata param596 if (body.contains("metadata")) {597 json metadata = json_value(body, "metadata", json::object());598 std::string user_id = json_value(metadata, "user_id", std::string());599 if (!user_id.empty()) {600 oai_body["__metadata_user_id"] = user_id;601 }602 }603 604 return oai_body;605}606 607json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff) {608 json delta = json::object();609 if (!diff.reasoning_content_delta.empty()) {610 delta["reasoning_content"] = diff.reasoning_content_delta;611 }612 if (!diff.content_delta.empty()) {613 delta["content"] = diff.content_delta;614 }615 if (diff.tool_call_index != std::string::npos) {616 json tool_call;617 tool_call["index"] = diff.tool_call_index;618 if (!diff.tool_call_delta.id.empty()) {619 tool_call["id"] = diff.tool_call_delta.id;620 tool_call["type"] = "function";621 }622 if (!diff.tool_call_delta.name.empty() || !diff.tool_call_delta.arguments.empty()) {623 json function = json::object();624 if (!diff.tool_call_delta.name.empty()) {625 function["name"] = diff.tool_call_delta.name;626 }627 if (!diff.tool_call_delta.arguments.empty()) {628 function["arguments"] = diff.tool_call_delta.arguments;629 }630 tool_call["function"] = function;631 }632 delta["tool_calls"] = json::array({ tool_call });633 }634 return delta;635}636 637json convert_transcriptions_to_chatcmpl(638 const json & inp_body,639 const common_chat_templates * tmpls,640 const std::map<std::string, uploaded_file> & in_files,641 std::vector<raw_buffer> & out_files) {642 // TODO @ngxson : this function may need to be improved in the future643 // handle input files644 out_files.clear();645 auto it = in_files.find("file");646 if (it != in_files.end()) {647 out_files.push_back(it->second.data);648 } else {649 throw std::invalid_argument("No input file found for transcription");650 }651 652 // handle input data653 std::string prompt = json_value(inp_body, "prompt", std::string());654 std::string language = json_value(inp_body, "language", std::string());655 std::string response_format = json_value(inp_body, "response_format", std::string("json"));656 if (response_format != "json") {657 throw std::invalid_argument("Only 'json' response_format is supported for transcription");658 }659 const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls);660 if (prompt.empty()) {661 prompt = preset.user;662 }663 if (!language.empty()) {664 prompt += string_format(" (language: %s)", language.c_str());665 }666 prompt += get_media_marker();667 668 json messages = json::array();669 if (!preset.system.empty()) {670 messages.push_back({{"role", "system"}, {"content", preset.system}});671 }672 messages.push_back({{"role", "user"}, {"content", prompt}});673 674 json chatcmpl_body = inp_body; // copy all fields675 chatcmpl_body["messages"] = messages;676 677 // because input from form-data, everything is string, we need to correct the types here678 std::string stream = json_value(inp_body, "stream", std::string("false"));679 chatcmpl_body["stream"] = stream == "true";680 681 if (inp_body.contains("max_tokens")) {682 std::string inp = inp_body["max_tokens"].get<std::string>();683 chatcmpl_body["max_tokens"] = std::stoul(inp);684 }685 686 if (inp_body.contains("temperature")) {687 std::string inp = inp_body["temperature"].get<std::string>();688 chatcmpl_body["temperature"] = std::stof(inp);689 }690 691 return chatcmpl_body;692}693