Felipe97/llama-cpp-compiled
01.1k
1#include "server-task.h"2 3#include "build-info.h"4#include "server-chat.h"5#include "chat.h"6#include "common.h"7#include "json-schema-to-grammar.h"8#include "llama.h"9#include "sampling.h"10#include "speculative.h"11#include "server-common.h"12 13#include <sstream>14 15//16// task_params17//18 19json task_params::format_logit_bias(const std::vector<llama_logit_bias> & logit_bias) const {20 json data = json::array();21 for (const auto & lb : logit_bias) {22 data.push_back(json{23 {"bias", lb.bias},24 {"token", lb.token},25 });26 }27 return data;28}29 30json task_params::to_json(bool only_metrics) const {31 std::vector<std::string> samplers;32 samplers.reserve(sampling.samplers.size());33 for (const auto & sampler : sampling.samplers) {34 samplers.emplace_back(common_sampler_type_to_str(sampler));35 }36 37 json lora = json::array();38 for (auto & it : this->lora) {39 lora.push_back({{"id", it.first}, {"scale", it.second}});40 }41 42 if (only_metrics) {43 return json {44 {"seed", sampling.seed},45 {"temperature", sampling.temp},46 {"dynatemp_range", sampling.dynatemp_range},47 {"dynatemp_exponent", sampling.dynatemp_exponent},48 {"top_k", sampling.top_k},49 {"top_p", sampling.top_p},50 {"min_p", sampling.min_p},51 {"top_n_sigma", sampling.top_n_sigma},52 {"xtc_probability", sampling.xtc_probability},53 {"xtc_threshold", sampling.xtc_threshold},54 {"typical_p", sampling.typ_p},55 {"repeat_last_n", sampling.penalty_last_n},56 {"repeat_penalty", sampling.penalty_repeat},57 {"presence_penalty", sampling.penalty_present},58 {"frequency_penalty", sampling.penalty_freq},59 {"dry_multiplier", sampling.dry_multiplier},60 {"dry_base", sampling.dry_base},61 {"dry_allowed_length", sampling.dry_allowed_length},62 {"dry_penalty_last_n", sampling.dry_penalty_last_n},63 {"mirostat", sampling.mirostat},64 {"mirostat_tau", sampling.mirostat_tau},65 {"mirostat_eta", sampling.mirostat_eta},66 {"adaptive_target", sampling.adaptive_target},67 {"adaptive_decay", sampling.adaptive_decay},68 {"max_tokens", n_predict},69 {"n_predict", n_predict}, // TODO: deduplicate?70 {"n_keep", n_keep},71 {"n_discard", n_discard},72 {"ignore_eos", sampling.ignore_eos},73 {"stream", stream},74 {"n_probs", sampling.n_probs},75 {"min_keep", sampling.min_keep},76 {"chat_format", common_chat_format_name(chat_parser_params.format)},77 {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},78 {"reasoning_in_content", chat_parser_params.reasoning_in_content},79 {"generation_prompt", chat_parser_params.generation_prompt},80 {"samplers", samplers},81 {"speculative.types", common_speculative_type_name_str(speculative.types)},82 {"timings_per_token", timings_per_token},83 {"post_sampling_probs", post_sampling_probs},84 {"backend_sampling", sampling.backend_sampling},85 {"lora", lora},86 };87 }88 89 auto grammar_triggers = json::array();90 for (const auto & trigger : sampling.grammar_triggers) {91 server_grammar_trigger ct(trigger);92 grammar_triggers.push_back(ct.to_json());93 }94 95 return json {96 {"seed", sampling.seed},97 {"temperature", sampling.temp},98 {"dynatemp_range", sampling.dynatemp_range},99 {"dynatemp_exponent", sampling.dynatemp_exponent},100 {"top_k", sampling.top_k},101 {"top_p", sampling.top_p},102 {"min_p", sampling.min_p},103 {"top_n_sigma", sampling.top_n_sigma},104 {"xtc_probability", sampling.xtc_probability},105 {"xtc_threshold", sampling.xtc_threshold},106 {"typical_p", sampling.typ_p},107 {"repeat_last_n", sampling.penalty_last_n},108 {"repeat_penalty", sampling.penalty_repeat},109 {"presence_penalty", sampling.penalty_present},110 {"frequency_penalty", sampling.penalty_freq},111 {"dry_multiplier", sampling.dry_multiplier},112 {"dry_base", sampling.dry_base},113 {"dry_allowed_length", sampling.dry_allowed_length},114 {"dry_penalty_last_n", sampling.dry_penalty_last_n},115 {"dry_sequence_breakers", sampling.dry_sequence_breakers},116 {"mirostat", sampling.mirostat},117 {"mirostat_tau", sampling.mirostat_tau},118 {"mirostat_eta", sampling.mirostat_eta},119 {"adaptive_target", sampling.adaptive_target},120 {"adaptive_decay", sampling.adaptive_decay},121 {"stop", antiprompt},122 {"max_tokens", n_predict},123 {"n_predict", n_predict}, // TODO: deduplicate?124 {"n_keep", n_keep},125 {"n_discard", n_discard},126 {"ignore_eos", sampling.ignore_eos},127 {"stream", stream},128 {"logit_bias", format_logit_bias(sampling.logit_bias)},129 {"n_probs", sampling.n_probs},130 {"min_keep", sampling.min_keep},131 {"grammar", common_grammar_value(sampling.grammar)},132 {"grammar_lazy", sampling.grammar_lazy},133 {"grammar_triggers", grammar_triggers},134 {"preserved_tokens", sampling.preserved_tokens},135 {"chat_format", common_chat_format_name(chat_parser_params.format)},136 {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},137 {"reasoning_in_content", chat_parser_params.reasoning_in_content},138 {"generation_prompt", chat_parser_params.generation_prompt},139 {"samplers", samplers},140 {"speculative.types", common_speculative_type_name_str(speculative.types)},141 {"timings_per_token", timings_per_token},142 {"post_sampling_probs", post_sampling_probs},143 {"backend_sampling", sampling.backend_sampling},144 {"lora", lora},145 };146}147 148//149// task_result_state150//151task_result_state::task_result_state(const common_chat_parser_params & chat_parser_params)152 : chat_parser_params(chat_parser_params)153 , oai_resp_id("resp_" + random_string())154 , oai_resp_reasoning_id("rs_" + random_string())155 , oai_resp_message_id("msg_" + random_string()) {156 if (chat_parser_params.is_continuation && !chat_parser_params.echo) {157 // initialize chat_msg to avoid emitting a delta containing the assistant prefill158 chat_msg = common_chat_parse("", true, chat_parser_params);159 }160}161 162common_chat_msg task_result_state::update_chat_msg(163 const std::string & text_added,164 bool is_partial,165 std::vector<common_chat_msg_diff> & diffs,166 bool filter_tool_calls) {167 generated_text += text_added;168 auto msg_prv_copy = chat_msg;169 //SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());170 auto new_msg = common_chat_parse(171 generated_text,172 is_partial,173 chat_parser_params);174 if (!new_msg.empty()) {175 new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id);176 chat_msg = new_msg;177 auto all_diffs = common_chat_msg_diff::compute_diffs(msg_prv_copy, chat_msg);178 179 if (!filter_tool_calls) {180 diffs = std::move(all_diffs);181 } else {182 for (auto & d : all_diffs) {183 // If this is a new type of delta, flush all currently pending tool call names184 for (size_t i = 0; i < chat_msg.tool_calls.size(); ++i) {185 if (sent_tool_call_names.count(i) || chat_msg.tool_calls[i].name.empty()) {186 continue;187 }188 if (d.tool_call_index != i || !d.tool_call_delta.arguments.empty()) {189 common_chat_msg_diff header;190 header.tool_call_index = i;191 header.tool_call_delta.id = chat_msg.tool_calls[i].id;192 header.tool_call_delta.name = chat_msg.tool_calls[i].name;193 diffs.push_back(std::move(header));194 sent_tool_call_names.insert(i);195 }196 }197 198 if (d.tool_call_index == std::string::npos) {199 diffs.push_back(std::move(d));200 } else {201 size_t i = d.tool_call_index;202 if (sent_tool_call_names.count(i)) {203 if (!d.tool_call_delta.arguments.empty()) {204 d.tool_call_delta.name = "";205 d.tool_call_delta.id = "";206 diffs.push_back(std::move(d));207 }208 } else {209 // Not sent yet.210 if (!d.tool_call_delta.arguments.empty() || !is_partial) {211 d.tool_call_delta.name = chat_msg.tool_calls[i].name;212 d.tool_call_delta.id = chat_msg.tool_calls[i].id;213 diffs.push_back(std::move(d));214 sent_tool_call_names.insert(i);215 } else {216 // Suppress217 }218 }219 }220 }221 // Final check at EOF222 if (!is_partial) {223 for (size_t i = 0; i < chat_msg.tool_calls.size(); ++i) {224 if (!sent_tool_call_names.count(i) && !chat_msg.tool_calls[i].name.empty()) {225 common_chat_msg_diff header;226 header.tool_call_index = i;227 header.tool_call_delta.id = chat_msg.tool_calls[i].id;228 header.tool_call_delta.name = chat_msg.tool_calls[i].name;229 diffs.push_back(std::move(header));230 sent_tool_call_names.insert(i);231 }232 }233 }234 }235 }236 return chat_msg;237}238 239//240// result_prompt_progress241//242json result_prompt_progress::to_json() const {243 return json {244 {"total", total},245 {"cache", cache},246 {"processed", processed},247 {"time_ms", time_ms},248 };249}250 251static inline std::string stop_type_to_str(stop_type type) {252 switch (type) {253 case STOP_TYPE_EOS: return "eos";254 case STOP_TYPE_WORD: return "word";255 case STOP_TYPE_LIMIT: return "limit";256 default: return "none";257 }258}259 260//261// completion_token_output262//263 264json completion_token_output::to_json(bool post_sampling_probs) const {265 json probs_for_token = json::array();266 for (const auto & p : probs) {267 std::string txt(p.txt);268 txt.resize(validate_utf8(txt));269 probs_for_token.push_back(json {270 {"id", p.tok},271 {"token", txt},272 {"bytes", str_to_bytes(p.txt)},273 {274 post_sampling_probs ? "prob" : "logprob",275 post_sampling_probs ? p.prob : logarithm(p.prob)276 },277 });278 }279 return probs_for_token;280}281 282json completion_token_output::probs_vector_to_json(const std::vector<completion_token_output> & probs, bool post_sampling_probs) {283 json out = json::array();284 for (const auto & p : probs) {285 std::string txt(p.text_to_send);286 txt.resize(validate_utf8(txt));287 out.push_back(json {288 {"id", p.tok},289 {"token", txt},290 {"bytes", str_to_bytes(p.text_to_send)},291 {292 post_sampling_probs ? "prob" : "logprob",293 post_sampling_probs ? p.prob : logarithm(p.prob)294 },295 {296 post_sampling_probs ? "top_probs" : "top_logprobs",297 p.to_json(post_sampling_probs)298 },299 });300 }301 return out;302}303 304float completion_token_output::logarithm(float x) {305 // the JSON library converts -inf to null, so we need to prevent that306 return x == 0.0f ? std::numeric_limits<float>::lowest() : std::log(x);307}308 309std::vector<unsigned char> completion_token_output::str_to_bytes(const std::string & str) {310 std::vector<unsigned char> bytes;311 for (unsigned char c : str) {312 bytes.push_back(c);313 }314 return bytes;315}316 317//318// server_task_result_cmpl_final319//320json server_task_result_cmpl_final::to_json() {321 GGML_ASSERT(is_updated && "update() must be called before to_json()");322 switch (res_type) {323 case TASK_RESPONSE_TYPE_NONE:324 return to_json_non_oaicompat();325 case TASK_RESPONSE_TYPE_OAI_CMPL:326 return to_json_oaicompat();327 case TASK_RESPONSE_TYPE_OAI_CHAT:328 return stream ? to_json_oaicompat_chat_stream() : to_json_oaicompat_chat();329 case TASK_RESPONSE_TYPE_OAI_RESP:330 return stream ? to_json_oaicompat_resp_stream() : to_json_oaicompat_resp();331 case TASK_RESPONSE_TYPE_OAI_ASR:332 return to_json_oaicompat_asr();333 case TASK_RESPONSE_TYPE_ANTHROPIC:334 return stream ? to_json_anthropic_stream() : to_json_anthropic();335 default:336 GGML_ASSERT(false && "Invalid task_response_type");337 }338}339 340json server_task_result_cmpl_final::to_json_non_oaicompat() {341 json res = json {342 {"index", index},343 {"content", content},344 {"tokens", tokens},345 {"id_slot", id_slot},346 {"stop", true},347 {"model", oaicompat_model},348 {"tokens_predicted", n_decoded},349 {"tokens_evaluated", n_prompt_tokens},350 {"generation_settings", generation_params.to_json()},351 {"prompt", prompt},352 {"has_new_line", has_new_line},353 {"truncated", truncated},354 {"stop_type", stop_type_to_str(stop)},355 {"stopping_word", stopping_word},356 {"tokens_cached", n_tokens_cached},357 {"timings", stats.to_json()},358 };359 if (!stream && !probs_output.empty()) {360 res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);361 }362 return response_fields.empty() ? res : json_get_nested_values(response_fields, res);363}364 365json server_task_result_cmpl_final::usage_json_oaicompat() {366 return json {367 {"completion_tokens", n_decoded},368 {"prompt_tokens", n_prompt_tokens},369 {"total_tokens", n_decoded + n_prompt_tokens},370 {"prompt_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},371 };372}373 374json server_task_result_cmpl_final::to_json_oaicompat() {375 std::time_t t = std::time(0);376 json logprobs = json(nullptr); // OAI default to null377 if (!stream && probs_output.size() > 0) {378 logprobs = json{379 {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},380 };381 }382 json finish_reason = "length";383 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {384 finish_reason = "stop";385 }386 json res = json {387 {"choices", json::array({388 json{389 {"text", content},390 {"index", index},391 {"logprobs", logprobs},392 {"finish_reason", finish_reason},393 }394 })},395 {"created", t},396 {"model", oaicompat_model},397 {"system_fingerprint", std::string(llama_build_info())},398 {"object", "text_completion"},399 {"usage", usage_json_oaicompat()},400 {"id", oaicompat_cmpl_id}401 };402 403 // extra fields for debugging purposes404 if (verbose) {405 res["__verbose"] = to_json_non_oaicompat();406 }407 if (stats.is_set()) {408 res["timings"] = stats.to_json();409 }410 411 return res;412}413 414json server_task_result_cmpl_final::to_json_oaicompat_chat() {415 std::string finish_reason = "length";416 common_chat_msg msg;417 if (!oaicompat_msg.empty()) {418 msg = oaicompat_msg;419 } else {420 msg.role = "assistant";421 msg.content = content;422 }423 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {424 finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";425 }426 427 json choice {428 {"finish_reason", finish_reason},429 {"index", index},430 {"message", msg.to_json_oaicompat()},431 };432 433 if (!stream && probs_output.size() > 0) {434 choice["logprobs"] = json{435 {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},436 };437 }438 439 std::time_t t = std::time(0);440 441 json res = json {442 {"choices", json::array({choice})},443 {"created", t},444 {"model", oaicompat_model},445 {"system_fingerprint", std::string(llama_build_info())},446 {"object", "chat.completion"},447 {"usage", usage_json_oaicompat()},448 {"id", oaicompat_cmpl_id}449 };450 451 // extra fields for debugging purposes452 if (verbose) {453 res["__verbose"] = to_json_non_oaicompat();454 }455 if (stats.is_set()) {456 res["timings"] = stats.to_json();457 }458 459 return res;460}461 462json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {463 std::time_t t = std::time(0);464 std::string finish_reason = "length";465 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {466 finish_reason = oaicompat_msg.tool_calls.empty() ? "stop" : "tool_calls";467 }468 469 json deltas = json::array();470 for (const auto & diff : oaicompat_msg_diffs) {471 deltas.push_back({472 {"choices", json::array({473 json {474 {"finish_reason", nullptr},475 {"index", index},476 {"delta", server_chat_msg_diff_to_json_oaicompat(diff)},477 },478 })},479 {"created", t},480 {"id", oaicompat_cmpl_id},481 {"model", oaicompat_model},482 {"system_fingerprint", std::string(llama_build_info())},483 {"object", "chat.completion.chunk"},484 });485 }486 487 deltas.push_back({488 {"choices", json::array({489 json {490 {"finish_reason", finish_reason},491 {"index", index},492 {"delta", json::object()},493 },494 })},495 {"created", t},496 {"id", oaicompat_cmpl_id},497 {"model", oaicompat_model},498 {"system_fingerprint", std::string(llama_build_info())},499 {"object", "chat.completion.chunk"},500 });501 502 if (include_usage) {503 // OpenAI API spec for chat.completion.chunks specifies an empty `choices` array for the last chunk when including usage504 // https://platform.openai.com/docs/api-reference/chat_streaming/streaming#chat_streaming/streaming-choices505 deltas.push_back({506 {"choices", json::array()},507 {"created", t},508 {"id", oaicompat_cmpl_id},509 {"model", oaicompat_model},510 {"system_fingerprint", std::string(llama_build_info())},511 {"object", "chat.completion.chunk"},512 {"usage", usage_json_oaicompat()},513 });514 }515 516 if (stats.is_set()) {517 deltas.back()["timings"] = stats.to_json();518 }519 520 // extra fields for debugging purposes521 if (verbose && !deltas.empty()) {522 deltas.front()["__verbose"] = to_json_non_oaicompat();523 }524 525 return deltas;526}527 528json server_task_result_cmpl_final::to_json_oaicompat_resp() {529 common_chat_msg msg;530 if (!oaicompat_msg.empty()) {531 msg = oaicompat_msg;532 } else {533 msg.role = "assistant";534 msg.content = content;535 }536 537 std::vector<json> output;538 539 if (msg.reasoning_content != "") {540 output.push_back(json {541 {"id", "rs_" + random_string()},542 {"summary", json::array()},543 {"type", "reasoning"},544 {"content", json::array({ json {545 {"text", msg.reasoning_content},546 {"type", "reasoning_text"},547 }})},548 {"encrypted_content", ""},549 {"status", "completed"},550 });551 }552 553 if (msg.content != "") {554 output.push_back(json {555 {"content", json::array({ json {556 {"type", "output_text"},557 {"annotations", json::array()},558 {"logprobs", json::array()},559 {"text", msg.content},560 }})},561 {"id", "msg_" + random_string()},562 {"role", msg.role},563 {"status", "completed"},564 {"type", "message"},565 });566 }567 568 for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) {569 output.push_back(json {570 {"id", "fc_" + tool_call.id},571 {"type", "function_call"},572 {"status", "completed"},573 {"arguments", tool_call.arguments},574 {"call_id", "call_" + tool_call.id},575 {"name", tool_call.name},576 });577 }578 579 std::time_t t = std::time(0);580 json res = {581 {"completed_at", t},582 {"created_at", t},583 {"id", oai_resp_id},584 {"model", oaicompat_model},585 {"object", "response"},586 {"output", output},587 {"status", "completed"},588 {"usage", json {589 {"input_tokens", n_prompt_tokens},590 {"output_tokens", n_decoded},591 {"total_tokens", n_decoded + n_prompt_tokens},592 {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},593 }},594 };595 596 return res;597}598 599json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {600 std::vector<json> server_sent_events;601 std::vector<json> output;602 603 if (oaicompat_msg.reasoning_content != "") {604 const json output_item = json {605 {"id", oai_resp_reasoning_id},606 {"summary", json::array()},607 {"type", "reasoning"},608 {"content", json::array({ json {609 {"text", oaicompat_msg.reasoning_content},610 {"type", "reasoning_text"},611 }})},612 {"encrypted_content", ""},613 };614 615 server_sent_events.push_back(json {616 {"event", "response.output_item.done"},617 {"data", json {618 {"type", "response.output_item.done"},619 {"item", output_item}620 }}621 });622 output.push_back(output_item);623 }624 625 if (oaicompat_msg.content != "") {626 server_sent_events.push_back(json {627 {"event", "response.output_text.done"},628 {"data", json {629 {"type", "response.output_text.done"},630 {"item_id", oai_resp_message_id},631 {"text", oaicompat_msg.content}632 }}633 });634 635 const json content_part = {636 {"type", "output_text"},637 {"annotations", json::array()},638 {"logprobs", json::array()},639 {"text", oaicompat_msg.content}640 };641 642 server_sent_events.push_back(json {643 {"event", "response.content_part.done"},644 {"data", json {645 {"type", "response.content_part.done"},646 {"item_id", oai_resp_message_id},647 {"part", content_part}648 }}649 });650 const json output_item = {651 {"type", "message"},652 {"status", "completed"},653 {"id", oai_resp_message_id},654 {"content", json::array({content_part})},655 {"role", "assistant"}656 };657 658 server_sent_events.push_back(json {659 {"event", "response.output_item.done"},660 {"data", json {661 {"type", "response.output_item.done"},662 {"item", output_item}663 }}664 });665 output.push_back(output_item);666 }667 668 for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) {669 const json output_item = {670 {"id", "fc_" + tool_call.id},671 {"type", "function_call"},672 {"status", "completed"},673 {"arguments", tool_call.arguments},674 {"call_id", "call_" + tool_call.id},675 {"name", tool_call.name}676 };677 server_sent_events.push_back(json {678 {"event", "response.output_item.done"},679 {"data", json {680 {"type", "response.output_item.done"},681 {"item", output_item}682 }}683 });684 output.push_back(output_item);685 }686 687 std::time_t t = std::time(0);688 server_sent_events.push_back(json {689 {"event", "response.completed"},690 {"data", json {691 {"type", "response.completed"},692 {"response", json {693 {"id", oai_resp_id},694 {"object", "response"},695 {"created_at", t},696 {"status", "completed"},697 {"model", oaicompat_model},698 {"output", output},699 {"usage", json {700 {"input_tokens", n_prompt_tokens},701 {"output_tokens", n_decoded},702 {"total_tokens", n_decoded + n_prompt_tokens},703 {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},704 }}705 }},706 }}707 });708 709 if (stats.is_set()) {710 server_sent_events.back().at("data")["timings"] = stats.to_json();711 }712 713 return server_sent_events;714}715 716json server_task_result_cmpl_final::to_json_oaicompat_asr() {717 json event = json {718 {"type", "transcript.text.done"},719 {"text", oaicompat_msg.content},720 {"usage", json {721 {"type", "tokens"},722 {"input_tokens", n_prompt_tokens},723 {"output_tokens", n_decoded},724 {"total_tokens", n_decoded + n_prompt_tokens},725 {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},726 }},727 };728 return event;729}730 731json server_task_result_cmpl_final::to_json_anthropic() {732 std::string stop_reason = "max_tokens";733 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {734 stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use";735 }736 737 json content_blocks = json::array();738 739 common_chat_msg msg;740 if (!oaicompat_msg.empty()) {741 msg = oaicompat_msg;742 } else {743 msg.role = "assistant";744 msg.content = content;745 }746 747 // thinking block comes first (Anthropic extended thinking format)748 if (!msg.reasoning_content.empty()) {749 content_blocks.push_back({750 {"type", "thinking"},751 {"thinking", msg.reasoning_content},752 {"signature", ""} // empty signature for local models (no cryptographic verification)753 });754 }755 756 if (!msg.content.empty()) {757 content_blocks.push_back({758 {"type", "text"},759 {"text", msg.content}760 });761 }762 763 for (const auto & tool_call : msg.tool_calls) {764 json tool_use_block = {765 {"type", "tool_use"},766 {"id", tool_call.id},767 {"name", tool_call.name}768 };769 770 try {771 tool_use_block["input"] = json::parse(tool_call.arguments);772 } catch (const std::exception &) {773 tool_use_block["input"] = json::object();774 }775 776 content_blocks.push_back(tool_use_block);777 }778 779 json res = {780 {"id", oaicompat_cmpl_id},781 {"type", "message"},782 {"role", "assistant"},783 {"content", content_blocks},784 {"model", oaicompat_model},785 {"stop_reason", stop_reason},786 {"stop_sequence", stopping_word.empty() ? nullptr : json(stopping_word)},787 {"usage", {788 {"cache_read_input_tokens", n_prompt_tokens_cache},789 {"input_tokens", n_prompt_tokens - n_prompt_tokens_cache},790 {"output_tokens", n_decoded}791 }}792 };793 794 return res;795}796 797json server_task_result_cmpl_final::to_json_anthropic_stream() {798 json events = json::array();799 800 std::string stop_reason = "max_tokens";801 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {802 stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use";803 }804 805 bool has_thinking = !oaicompat_msg.reasoning_content.empty();806 bool has_text = !oaicompat_msg.content.empty();807 size_t num_tool_calls = oaicompat_msg.tool_calls.size();808 809 // content block indices: thinking (0) -> text (0 or 1) -> tool_use (n+)810 size_t thinking_block_index = 0;811 size_t text_block_index = has_thinking ? 1 : 0;812 813 bool thinking_block_started = false;814 bool text_block_started = false;815 std::unordered_set<size_t> tool_calls_started;816 817 for (const auto & diff : oaicompat_msg_diffs) {818 // handle thinking/reasoning content819 if (!diff.reasoning_content_delta.empty()) {820 if (!thinking_block_started) {821 events.push_back({822 {"event", "content_block_start"},823 {"data", {824 {"type", "content_block_start"},825 {"index", thinking_block_index},826 {"content_block", {827 {"type", "thinking"},828 {"thinking", ""}829 }}830 }}831 });832 thinking_block_started = true;833 }834 835 events.push_back({836 {"event", "content_block_delta"},837 {"data", {838 {"type", "content_block_delta"},839 {"index", thinking_block_index},840 {"delta", {841 {"type", "thinking_delta"},842 {"thinking", diff.reasoning_content_delta}843 }}844 }}845 });846 }847 848 // handle regular text content849 if (!diff.content_delta.empty()) {850 if (!text_block_started) {851 events.push_back({852 {"event", "content_block_start"},853 {"data", {854 {"type", "content_block_start"},855 {"index", text_block_index},856 {"content_block", {857 {"type", "text"},858 {"text", ""}859 }}860 }}861 });862 text_block_started = true;863 }864 865 events.push_back({866 {"event", "content_block_delta"},867 {"data", {868 {"type", "content_block_delta"},869 {"index", text_block_index},870 {"delta", {871 {"type", "text_delta"},872 {"text", diff.content_delta}873 }}874 }}875 });876 }877 878 // handle tool calls879 if (diff.tool_call_index != std::string::npos) {880 size_t content_block_index = (has_thinking ? 1 : 0) + (has_text ? 1 : 0) + diff.tool_call_index;881 882 if (tool_calls_started.find(diff.tool_call_index) == tool_calls_started.end()) {883 const auto & full_tool_call = oaicompat_msg.tool_calls[diff.tool_call_index];884 885 events.push_back({886 {"event", "content_block_start"},887 {"data", {888 {"type", "content_block_start"},889 {"index", content_block_index},890 {"content_block", {891 {"type", "tool_use"},892 {"id", full_tool_call.id},893 {"name", full_tool_call.name}894 }}895 }}896 });897 tool_calls_started.insert(diff.tool_call_index);898 }899 900 if (!diff.tool_call_delta.arguments.empty()) {901 events.push_back({902 {"event", "content_block_delta"},903 {"data", {904 {"type", "content_block_delta"},905 {"index", content_block_index},906 {"delta", {907 {"type", "input_json_delta"},908 {"partial_json", diff.tool_call_delta.arguments}909 }}910 }}911 });912 }913 }914 }915 916 // close content blocks in order917 if (has_thinking) {918 // Anthropic API requires a signature_delta before closing thinking blocks919 // We use an empty signature since we can't generate a cryptographic signature for local models920 events.push_back({921 {"event", "content_block_delta"},922 {"data", {923 {"type", "content_block_delta"},924 {"index", thinking_block_index},925 {"delta", {926 {"type", "signature_delta"},927 {"signature", ""}928 }}929 }}930 });931 events.push_back({932 {"event", "content_block_stop"},933 {"data", {934 {"type", "content_block_stop"},935 {"index", thinking_block_index}936 }}937 });938 }939 940 if (has_text) {941 events.push_back({942 {"event", "content_block_stop"},943 {"data", {944 {"type", "content_block_stop"},945 {"index", text_block_index}946 }}947 });948 }949 950 for (size_t i = 0; i < num_tool_calls; i++) {951 size_t content_block_index = (has_thinking ? 1 : 0) + (has_text ? 1 : 0) + i;952 events.push_back({953 {"event", "content_block_stop"},954 {"data", {955 {"type", "content_block_stop"},956 {"index", content_block_index}957 }}958 });959 }960 961 events.push_back({962 {"event", "message_delta"},963 {"data", {964 {"type", "message_delta"},965 {"delta", {966 {"stop_reason", stop_reason},967 {"stop_sequence", stopping_word.empty() ? nullptr : json(stopping_word)}968 }},969 {"usage", {970 {"output_tokens", n_decoded}971 }}972 }}973 });974 975 events.push_back({976 {"event", "message_stop"},977 {"data", {978 {"type", "message_stop"}979 }}980 });981 982 return events;983}984 985//986// server_task_result_cmpl_partial987//988void server_task_result_cmpl_partial::update(task_result_state & state) {989 is_updated = true;990 if (is_begin) {991 return; // begin marker only flushes headers, skip parsing992 }993 state.update_chat_msg(content, true, oaicompat_msg_diffs);994 995 // Copy current state for use in to_json_*() (reflects state BEFORE this chunk)996 thinking_block_started = state.thinking_block_started;997 text_block_started = state.text_block_started;998 999 oai_resp_created = state.oai_resp_created;1000 oai_resp_id = state.oai_resp_id;1001 oai_resp_reasoning_id = state.oai_resp_reasoning_id;1002 oai_resp_message_id = state.oai_resp_message_id;1003 oai_resp_fc_id = state.oai_resp_fc_id;1004 1005 // track if the accumulated message has any reasoning content1006 anthropic_has_reasoning = !state.chat_msg.reasoning_content.empty();1007 1008 if (res_type == TASK_RESPONSE_TYPE_OAI_RESP && !state.oai_resp_created && (is_progress || n_decoded == 1)) {1009 state.oai_resp_created = true;1010 }1011 1012 // Pre-compute state updates based on diffs (for next chunk)1013 for (const common_chat_msg_diff & diff : oaicompat_msg_diffs) {1014 if (!diff.reasoning_content_delta.empty() && !state.thinking_block_started) {1015 state.thinking_block_started = true;1016 }1017 if (!diff.content_delta.empty() && !state.text_block_started) {1018 state.text_block_started = true;1019 }1020 if (!diff.tool_call_delta.name.empty()) {1021 state.oai_resp_fc_id = diff.tool_call_delta.id;1022 }1023 }1024}1025 1026json server_task_result_cmpl_partial::to_json() {1027 GGML_ASSERT(is_updated && "update() must be called before to_json()");1028 if (is_begin) {1029 return nullptr; // simply signal to HTTP handler to send the headers and status code1030 }1031 switch (res_type) {1032 case TASK_RESPONSE_TYPE_NONE:1033 return to_json_non_oaicompat();1034 case TASK_RESPONSE_TYPE_OAI_CMPL:1035 return to_json_oaicompat();1036 case TASK_RESPONSE_TYPE_OAI_CHAT:1037 return to_json_oaicompat_chat();1038 case TASK_RESPONSE_TYPE_OAI_RESP:1039 return to_json_oaicompat_resp();1040 case TASK_RESPONSE_TYPE_OAI_ASR:1041 return to_json_oaicompat_asr();1042 case TASK_RESPONSE_TYPE_ANTHROPIC:1043 return to_json_anthropic();1044 default:1045 GGML_ASSERT(false && "Invalid task_response_type");1046 }1047}1048 1049json server_task_result_cmpl_partial::to_json_non_oaicompat() {1050 // non-OAI-compat JSON1051 json res = json {1052 {"index", index},1053 {"content", content},1054 {"tokens", tokens},1055 {"stop", false},1056 {"id_slot", id_slot},1057 {"tokens_predicted", n_decoded},1058 {"tokens_evaluated", n_prompt_tokens},1059 };1060 // populate the timings object when needed (usually for the last response or with timings_per_token enabled)1061 if (stats.is_set()) {1062 res["timings"] = stats.to_json();1063 }1064 if (is_progress) {1065 res["prompt_progress"] = progress.to_json();1066 }1067 if (!prob_output.probs.empty()) {1068 res["completion_probabilities"] = completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs);1069 }1070 return res;1071}1072 1073json server_task_result_cmpl_partial::to_json_oaicompat() {1074 std::time_t t = std::time(0);1075 json logprobs = json(nullptr); // OAI default to null1076 if (prob_output.probs.size() > 0) {1077 logprobs = json{1078 {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)},1079 };1080 }1081 json res = json {1082 {"choices", json::array({1083 json{1084 {"text", content},1085 {"index", index},1086 {"logprobs", logprobs},1087 {"finish_reason", nullptr},1088 }1089 })},1090 {"created", t},1091 {"model", oaicompat_model},1092 {"system_fingerprint", std::string(llama_build_info())},1093 {"object", "text_completion"},1094 {"id", oaicompat_cmpl_id}1095 };1096 1097 // extra fields for debugging purposes1098 if (verbose) {1099 res["__verbose"] = to_json_non_oaicompat();1100 }1101 if (stats.is_set()) {1102 res["timings"] = stats.to_json();1103 }1104 if (is_progress) {1105 res["prompt_progress"] = progress.to_json();1106 }1107 1108 return res;1109}1110 1111json server_task_result_cmpl_partial::to_json_oaicompat_chat() {1112 bool first = n_decoded == 1;1113 std::time_t t = std::time(0);1114 json choices;1115 1116 std::vector<json> deltas;1117 auto add_delta = [&](const json & delta) {1118 deltas.push_back({1119 {"choices", json::array({1120 json {1121 {"finish_reason", nullptr},1122 {"index", index},1123 {"delta", delta},1124 },1125 })},1126 {"created", t},1127 {"id", oaicompat_cmpl_id},1128 {"model", oaicompat_model},1129 {"system_fingerprint", std::string(llama_build_info())},1130 {"object", "chat.completion.chunk"},1131 });1132 };1133 // We have to send an initial update to conform to openai behavior1134 if (first || is_progress) {1135 add_delta({1136 {"role", "assistant"},1137 {"content", nullptr},1138 });1139 }1140 1141 for (const auto & diff : oaicompat_msg_diffs) {1142 add_delta(server_chat_msg_diff_to_json_oaicompat(diff));1143 }1144 1145 if (!deltas.empty()) {1146 auto & last_json = deltas[deltas.size() - 1];1147 GGML_ASSERT(last_json.at("choices").size() >= 1);1148 1149 if (prob_output.probs.size() > 0) {1150 last_json.at("choices").at(0)["logprobs"] = json {1151 {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)},1152 };1153 }1154 1155 if (stats.is_set()) {1156 last_json["timings"] = stats.to_json();1157 }1158 if (is_progress) {1159 last_json["prompt_progress"] = progress.to_json();1160 }1161 }1162 1163 return deltas;1164}1165 1166json server_task_result_cmpl_partial::to_json_oaicompat_resp() {1167 std::vector<json> events;1168 1169 if (!oai_resp_created) {1170 events.push_back(json {1171 {"event", "response.created"},1172 {"data", json {1173 {"type", "response.created"},1174 {"response", json {1175 {"id", oai_resp_id},1176 {"object", "response"},1177 {"status", "in_progress"},1178 }},1179 }},1180 });1181 events.push_back(json {1182 {"event", "response.in_progress"},1183 {"data", json {1184 {"type", "response.in_progress"},1185 {"response", json {1186 {"id", oai_resp_id},1187 {"object", "response"},1188 {"status", "in_progress"},1189 }},1190 }},1191 });1192 } else if (is_progress) {1193 events.push_back(json {1194 {"event", "response.in_progress"},1195 {"data", json {1196 {"type", "response.in_progress"},1197 {"response", json {1198 {"id", oai_resp_id},1199 {"object", "response"},1200 {"status", "in_progress"},