Felipe97/llama-cpp-compiled
01.1k
1#include "chat.h"2 3#include "chat-auto-parser-helpers.h"4#include "chat-auto-parser.h"5#include "chat-peg-parser.h"6#include "common.h"7#include "ggml.h"8#include "json-schema-to-grammar.h"9#include "json.h"10#include "log.h"11#include "parsers/parsers.h"12 13#include "jinja/value.h"14#include "jinja/runtime.h"15#include "jinja/caps.h"16#include "peg-parser.h"17 18#include <algorithm>19#include <cstdio>20#include <cstdlib>21#include <ctime>22#include <exception>23#include <functional>24#include <iomanip>25#include <map>26 27#include <optional>28#include <sstream>29#include <stdexcept>30#include <string>31#include <utility>32#include <vector>33 34using json = common_json;35 36static std::string format_time(const std::chrono::system_clock::time_point & now, const std::string & format) {37 auto time = std::chrono::system_clock::to_time_t(now);38 auto local_time = *std::localtime(&time);39 std::ostringstream ss;40 ss << std::put_time(&local_time, format.c_str());41 auto res = ss.str();42 return res;43}44 45static json safe_args_parse(const std::string & to_parse) {46 std::string stripped = to_parse;47 if (to_parse.at(0) == '"' && to_parse.at(to_parse.length() - 1) == '"') {48 stripped = to_parse.substr(1, to_parse.length() - 1);49 }50 try {51 return json::parse(stripped);52 } catch (const common_json_error & e) {53 return stripped;54 }55}56 57static std::string string_diff(const std::string & last, const std::string & current) {58 if (last.empty()) {59 return current;60 }61 if (!string_starts_with(current, last)) {62 if (string_starts_with(last, current)) {63 // This happens if the last generation ended on a partial stop word (not erased),64 // and the current ended on a stop word (erased).65 return "";66 }67 throw std::runtime_error("Invalid diff: '" + last + "' not found at start of '" + current + "'");68 }69 return current.substr(last.size());70}71 72static bool has_content_or_tool_calls(const common_chat_msg & msg) {73 return !msg.content.empty() || !msg.tool_calls.empty();74}75 76std::string common_chat_msg::render_content(const std::string & delimiter) const {77 if (!content.empty() && !content_parts.empty()) {78 throw std::runtime_error("Cannot specify both content and content_parts");79 }80 if (!content.empty()) {81 return content;82 }83 84 std::string text;85 for (const auto & part : content_parts) {86 if (part.type == "text") {87 if (!text.empty()) {88 text += delimiter;89 }90 text += part.text;91 }92 }93 return text;94}95 96common_chat_role common_chat_role_from_string(const std::string & role) {97 if (role == "system") { return COMMON_CHAT_ROLE_SYSTEM; }98 if (role == "assistant") { return COMMON_CHAT_ROLE_ASSISTANT; }99 if (role == "user") { return COMMON_CHAT_ROLE_USER; }100 if (role == "tool") { return COMMON_CHAT_ROLE_TOOL; }101 return COMMON_CHAT_ROLE_UNKNOWN;102}103 104const char * common_chat_role_to_string(common_chat_role role) {105 switch (role) {106 case COMMON_CHAT_ROLE_SYSTEM: return "system";107 case COMMON_CHAT_ROLE_ASSISTANT: return "assistant";108 case COMMON_CHAT_ROLE_USER: return "user";109 case COMMON_CHAT_ROLE_TOOL: return "tool";110 case COMMON_CHAT_ROLE_UNKNOWN: return "";111 }112 return "";113}114 115json common_chat_msg_delimiters::to_json() const {116 json result = json::array();117 for (const auto & d : delimiters) {118 result.push_back({119 { "role", common_chat_role_to_string(d.role) },120 { "delimiter", d.delimiter },121 });122 }123 return result;124}125 126common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) {127 common_chat_msg_delimiters result;128 129 if (!delimiters.is_array()) {130 return result;131 }132 133 result.delimiters.reserve(delimiters.size());134 for (const auto & d : delimiters) {135 if (!d.is_object()) {136 continue;137 }138 result.delimiters.push_back({139 common_chat_role_from_string(d.value("role", std::string())),140 d.value("delimiter", std::string()),141 });142 }143 144 return result;145}146 147void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) {148 for (auto & d : delimiters) {149 d.tokens = common_tokenize(vocab, d.delimiter, false, true);150 }151}152 153common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips) const {154 std::vector<std::pair<common_chat_role, size_t>> matches;155 156 auto skip = skips.begin();157 for (size_t i = 0; i < tokens.size();) {158 if (skip != skips.end() && i == skip->first) {159 i += skip->second;160 ++skip;161 continue;162 }163 for (const auto & d : delimiters) {164 if (i + d.tokens.size() > tokens.size()) {165 continue;166 }167 if (std::equal(d.tokens.begin(), d.tokens.end(), tokens.begin() + i)) {168 matches.emplace_back(d.role, i);169 break;170 }171 }172 i++;173 }174 175 matches.emplace_back(COMMON_CHAT_ROLE_UNKNOWN, tokens.size());176 177 common_chat_msg_spans spans;178 for (size_t i = 0; i + 1 < matches.size(); i++) {179 const auto & curr = matches[i];180 const auto & next = matches[i + 1];181 spans.add(curr.first, curr.second, next.second - curr.second);182 }183 184 return spans;185}186 187json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {188 if (!content.empty() && !content_parts.empty()) {189 throw std::runtime_error("Cannot specify both content and content_parts");190 }191 json jmsg {192 {"role", role},193 };194 if (!content.empty()) {195 jmsg["content"] = content;196 } else if (!content_parts.empty()) {197 if (concat_typed_text || contains_media()) {198 std::string text;199 bool last_was_media_marker = false;200 // join parts with newline, do not add newline before or after media markers201 for (const auto & part : content_parts) {202 bool add_new_line = true;203 if (part.type == "text") {204 add_new_line = !last_was_media_marker && !text.empty();205 last_was_media_marker = false;206 } else if (part.type == "media_marker") {207 add_new_line = false;208 last_was_media_marker = true;209 } else {210 LOG_WRN("Ignoring content part type: %s\n", part.type.c_str());211 continue;212 }213 214 if (add_new_line) {215 text += '\n';216 }217 218 text += part.text;219 }220 jmsg["content"] = text;221 } else {222 auto & parts = jmsg["content"] = json::array();223 for (const auto & part : content_parts) {224 parts.push_back({225 {"type", part.type},226 {"text", part.text},227 });228 }229 }230 } else {231 jmsg["content"] = "";232 }233 if (!reasoning_content.empty()) {234 jmsg["reasoning_content"] = reasoning_content;235 }236 if (!tool_name.empty()) {237 jmsg["name"] = tool_name;238 }239 if (!tool_call_id.empty()) {240 jmsg["tool_call_id"] = tool_call_id;241 }242 if (!tool_calls.empty()) {243 jmsg["tool_calls"] = json::array();244 auto & jtool_calls = jmsg["tool_calls"];245 for (const auto & tool_call : tool_calls) {246 json tc {247 {"type", "function"},248 {"function", {249 {"name", tool_call.name},250 {"arguments", json(tool_call.arguments)},251 }},252 };253 if (!tool_call.id.empty()) {254 tc["id"] = tool_call.id;255 }256 // Some templates generate and require an id (sometimes in a very specific format, e.g. Mistral Nemo).257 // We only generate a random id for the ones that don't generate one by themselves258 // (they also won't get to see it as their template likely doesn't use it, so it's all for the client)259 // {"id", tc.id.empty() ? gen_tool_call_id() : tc.id},260 jtool_calls.push_back(tc);261 }262 }263 264 return jmsg;265}266 267std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const common_chat_msg & msg_prv,268 const common_chat_msg & msg_new) {269 std::vector<common_chat_msg_diff> diffs;270 if (msg_new.tool_calls.size() > msg_prv.tool_calls.size()) {271 diffs.reserve(msg_new.tool_calls.size() - msg_prv.tool_calls.size() + 3);272 } else {273 diffs.reserve(3);274 }275 276 // TODO: these can become expensive for long messages - how to optimize?277 if (msg_prv.reasoning_content != msg_new.reasoning_content) {278 auto & diff = diffs.emplace_back();279 diff.reasoning_content_delta = string_diff(msg_prv.reasoning_content, msg_new.reasoning_content);280 }281 if (msg_prv.content != msg_new.content) {282 auto & diff = diffs.emplace_back();283 diff.content_delta = string_diff(msg_prv.content, msg_new.content);284 }285 286 if (msg_new.tool_calls.size() < msg_prv.tool_calls.size()) {287 std::string err = "Invalid diff: now finding less tool calls!\n";288 err += " Previous (" + std::to_string(msg_prv.tool_calls.size()) + "):\n";289 for (const auto & tc : msg_prv.tool_calls) {290 err += " - name: '" + tc.name + "', args: '" + tc.arguments + "'\n";291 }292 err += " Current (" + std::to_string(msg_new.tool_calls.size()) + "):\n";293 for (const auto & tc : msg_new.tool_calls) {294 err += " - name: '" + tc.name + "', args: '" + tc.arguments + "'\n";295 }296 err += " Current msg text content:\n" + msg_new.content + "\n";297 throw std::runtime_error(err);298 }299 300 if (!msg_prv.tool_calls.empty()) {301 const auto idx = msg_prv.tool_calls.size() - 1;302 const auto & pref = msg_prv.tool_calls[idx];303 const auto & newf = msg_new.tool_calls[idx];304 // Allow tool name to change during incremental parsing:305 // - empty -> non-empty (initial discovery)306 // - prefix -> longer string (name grows as more input is parsed)307 if (pref.name != newf.name && !pref.name.empty() && !newf.name.empty()) {308 // Check if one is a prefix of the other (for incremental parsing where names grow or shrink)309 bool is_prefix = (newf.name.rfind(pref.name, 0) == 0);310 if (!is_prefix) {311 LOG_ERR("Tool call mismatch: prev='%s' new='%s'\n", pref.name.c_str(), newf.name.c_str());312 throw std::runtime_error("Invalid diff: tool call mismatch!");313 }314 }315 const auto args_diff = string_diff(pref.arguments, newf.arguments);316 if (!args_diff.empty() || pref.id != newf.id || pref.name != newf.name) {317 auto & diff = diffs.emplace_back();318 diff.tool_call_index = idx;319 if (pref.id != newf.id || pref.name != newf.name) {320 diff.tool_call_delta.id = newf.id;321 diff.tool_call_delta.name = newf.name;322 }323 diff.tool_call_delta.arguments = args_diff;324 }325 }326 for (size_t idx = msg_prv.tool_calls.size(); idx < msg_new.tool_calls.size(); ++idx) {327 auto & diff = diffs.emplace_back();328 diff.tool_call_index = idx;329 diff.tool_call_delta = msg_new.tool_calls[idx];330 }331 332 return diffs;333}334 335using chat_template_caps = jinja::caps;336 337struct common_chat_templates {338 bool add_bos;339 bool add_eos;340 bool has_explicit_template; // Model had builtin template or template overridden was specified.341 std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)342 std::unique_ptr<common_chat_template> template_tool_use;343};344 345common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {346 if (tool_choice == "auto") {347 return COMMON_CHAT_TOOL_CHOICE_AUTO;348 }349 if (tool_choice == "none") {350 return COMMON_CHAT_TOOL_CHOICE_NONE;351 }352 if (tool_choice == "required") {353 return COMMON_CHAT_TOOL_CHOICE_REQUIRED;354 }355 throw std::invalid_argument("Invalid tool_choice: " + tool_choice);356}357 358bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates) {359 common_chat_templates_inputs inputs;360 inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;361 common_chat_msg msg;362 msg.role = "user";363 msg.content = "test";364 inputs.messages = { msg };365 inputs.enable_thinking = true;366 inputs.add_generation_prompt = true;367 inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;368 369 auto params = common_chat_templates_apply(chat_templates, inputs);370 return params.supports_thinking;371}372 373std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messages) {374 std::vector<common_chat_msg> msgs;375 376 try {377 if (!messages.is_array()) {378 throw std::invalid_argument("Expected 'messages' to be an array, got " + messages.dump());379 }380 381 for (const auto & message : messages) {382 if (!message.is_object()) {383 throw std::invalid_argument("Expected 'message' to be an object, got " + message.dump());384 }385 386 common_chat_msg msg;387 if (!message.contains("role")) {388 throw std::invalid_argument("Missing 'role' in message: " + message.dump());389 }390 msg.role = message.at("role");391 392 auto has_content = message.contains("content");393 auto has_tool_calls = message.contains("tool_calls");394 if (has_content) {395 const auto & content = message.at("content");396 if (content.is_string()) {397 msg.content = content;398 } else if (content.is_array()) {399 for (const auto & part : content) {400 if (!part.contains("type")) {401 throw std::invalid_argument("Missing content part type: " + part.dump());402 }403 const auto & type = part.at("type");404 if (type != "text" && type != "media_marker") {405 throw std::invalid_argument("Unsupported content part type: " + type.dump());406 }407 common_chat_msg_content_part msg_part;408 msg_part.type = type;409 msg_part.text = part.at("text");410 msg.content_parts.push_back(msg_part);411 }412 } else if (!content.is_null()) {413 throw std::invalid_argument("Invalid 'content' type: expected string or array, got " +414 content.dump() +415 " (ref: https://github.com/ggml-org/llama.cpp/issues/8367)");416 }417 }418 if (has_tool_calls) {419 for (const auto & tool_call : message.at("tool_calls")) {420 common_chat_tool_call tc;421 if (!tool_call.contains("type")) {422 throw std::invalid_argument("Missing tool call type: " + tool_call.dump());423 }424 const auto & type = tool_call.at("type");425 if (type != "function") {426 throw std::invalid_argument("Unsupported tool call type: " + tool_call.dump());427 }428 if (!tool_call.contains("function")) {429 throw std::invalid_argument("Missing tool call function: " + tool_call.dump());430 }431 const auto & fc = tool_call.at("function");432 if (!fc.contains("name")) {433 throw std::invalid_argument("Missing tool call name: " + tool_call.dump());434 }435 tc.name = fc.at("name");436 const auto & args = fc.at("arguments");437 if (args.is_string()) {438 tc.arguments = args;439 } else {440 tc.arguments = args.dump();441 }442 if (tool_call.contains("id")) {443 tc.id = tool_call.at("id");444 }445 msg.tool_calls.push_back(tc);446 }447 }448 if (!has_content && !has_tool_calls) {449 throw std::invalid_argument(450 "Expected 'content' or 'tool_calls' (ref: https://github.com/ggml-org/llama.cpp/issues/8367 & "451 "https://github.com/ggml-org/llama.cpp/issues/12279)");452 }453 if (message.contains("reasoning_content")) {454 msg.reasoning_content = message.at("reasoning_content");455 }456 if (message.contains("name")) {457 msg.tool_name = message.at("name");458 }459 if (message.contains("tool_call_id")) {460 msg.tool_call_id = message.at("tool_call_id");461 }462 463 msgs.push_back(msg);464 }465 } catch (const std::exception & e) {466 // @ngxson : disable otherwise it's bloating the API response467 // printf("%s\n", std::string("; messages = ") + messages.dump(2));468 throw std::runtime_error("Failed to parse messages: " + std::string(e.what()));469 }470 471 return msgs;472}473 474struct messages_inp_normalizer {475 const jinja::caps & caps;476 477 messages_inp_normalizer(const jinja::caps & c) : caps(c) {}478 479 // handle supports_string_content / supports_typed_content480 // if string=true and array=false, convert array to string481 // if string=false and array=true, convert string to array482 // if both are true, do nothing483 json normalize(const json & messages) {484 bool only_string = caps.supports_string_content && !caps.supports_typed_content;485 bool only_typed = !caps.supports_string_content && caps.supports_typed_content;486 if ((!only_string && !only_typed) || !messages.is_array()) {487 return messages;488 }489 json normalized = json::array();490 for (const auto & msg : messages) {491 json copy = msg;492 if (copy.contains("content")) {493 json & it = copy.at("content");494 if (only_typed && it.is_string()) {495 it = json::array({496 json{497 {"type", "text"},498 {"text", it.get<std::string>()},499 }500 });501 } else if (only_string && it.is_array()) {502 it = concat_content_parts(it);503 }504 }505 normalized.push_back(std::move(copy));506 }507 return normalized;508 }509 510 // join parts with newline, do not add newline before or after media markers511 static std::string concat_content_parts(const json & parts) {512 std::string text;513 bool last_was_media_marker = false;514 for (const auto & part : parts) {515 std::string type = part.value("type", "");516 bool add_new_line = true;517 if (type == "text") {518 add_new_line = !last_was_media_marker && !text.empty();519 last_was_media_marker = false;520 } else if (type == "media_marker") {521 add_new_line = false;522 last_was_media_marker = true;523 } else {524 LOG_WRN("Ignoring content part type: %s\n", type.c_str());525 continue;526 }527 528 if (add_new_line) {529 text += '\n';530 }531 532 text += part.value("text", "");533 }534 return text;535 }536};537 538static json render_message_to_json(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {539 if (!c.supports_string_content && !c.supports_typed_content) {540 LOG_WRN("%s: Neither string content nor typed content is supported by the template. This is unexpected and may lead to issues.\n", __func__);541 }542 543 json messages = json::array();544 for (const auto & msg : msgs) {545 messages.push_back(msg.to_json_oaicompat(/* concat_typed_text= */ false));546 }547 return messages_inp_normalizer(c).normalize(messages);548}549 550// DEPRECATED: only used in tests551json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {552 jinja::caps c;553 c.supports_string_content = true;554 c.supports_typed_content = !concat_typed_text;555 return render_message_to_json(msgs, c);556}557 558json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools) {559 if (tools.empty()) {560 return json();561 }562 563 auto result = json::array();564 for (const auto & tool : tools) {565 result.push_back({566 { "type", "function" },567 { "function", {568 { "name", tool.name },569 { "description", tool.description },570 { "parameters", json::parse(tool.parameters) },571 }},572 });573 }574 return result;575}576 577json common_chat_tool_parameters(const json & function) {578 if (function.contains("parameters")) {579 const auto & params = function.at("parameters");580 if (!params.is_null() && !(params.is_object() && params.empty())) {581 return params;582 }583 }584 return json{{"type", "object"}, {"properties", json::object()}};585}586 587std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & tools) {588 std::vector<common_chat_tool> result;589 590 try {591 if (!tools.is_null()) {592 if (!tools.is_array()) {593 throw std::invalid_argument("Expected 'tools' to be an array, got " + tools.dump());594 }595 for (const auto & tool : tools) {596 if (!tool.contains("type")) {597 throw std::invalid_argument("Missing tool type: " + tool.dump());598 }599 const auto & type = tool.at("type");600 if (!type.is_string() || type != "function") {601 throw std::invalid_argument("Unsupported tool type: " + tool.dump());602 }603 if (!tool.contains("function")) {604 throw std::invalid_argument("Missing tool function: " + tool.dump());605 }606 607 const auto & function = tool.at("function");608 result.push_back({609 /* .name = */ function.at("name"),610 /* .description = */ function.value("description", ""),611 /* .parameters = */ function.value("parameters", json::object()).dump(),612 });613 }614 }615 } catch (const std::exception & e) {616 throw std::runtime_error("Failed to parse tools: " + std::string(e.what()) + "; tools = " + tools.dump(2));617 }618 619 return result;620}621 622common_chat_continuation common_chat_continuation_parse(const common_json & value) {623 if (value.is_boolean() && value.get<bool>()) {624 return COMMON_CHAT_CONTINUATION_AUTO;625 }626 if (value.is_string()) {627 auto value_str = value.get<std::string>();628 if (value_str == "reasoning_content") {629 return COMMON_CHAT_CONTINUATION_REASONING;630 }631 if (value_str == "content") {632 return COMMON_CHAT_CONTINUATION_CONTENT;633 }634 }635 return COMMON_CHAT_CONTINUATION_NONE;636}637 638bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {639 if (use_jinja) {640 try {641 common_chat_msg msg;642 msg.role = "user";643 msg.content = "test";644 645 auto tmpls = common_chat_templates_init(/* model= */ nullptr, tmpl);646 647 common_chat_templates_inputs inputs;648 inputs.messages = { msg };649 650 common_chat_templates_apply(tmpls.get(), inputs);651 return true;652 } catch (const std::exception & e) {653 LOG_ERR("%s: failed to apply template: %s\n", __func__, e.what());654 return false;655 }656 }657 llama_chat_message chat[] = {658 { "user", "test" }659 };660 const int res = llama_chat_apply_template(tmpl.c_str(), chat, 1, true, nullptr, 0);661 return res >= 0;662}663 664std::string common_chat_format_single(const struct common_chat_templates * tmpls,665 const std::vector<common_chat_msg> & past_msg,666 const common_chat_msg & new_msg,667 bool add_ass,668 bool use_jinja) {669 common_chat_templates_inputs inputs;670 inputs.use_jinja = use_jinja;671 inputs.add_bos = tmpls->add_bos;672 inputs.add_eos = tmpls->add_eos;673 674 std::string fmt_past_msg;675 if (!past_msg.empty()) {676 inputs.messages = past_msg;677 inputs.add_generation_prompt = false;678 fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt;679 }680 std::ostringstream ss;681 // if the past_msg ends with a newline, we must preserve it in the formatted version682 if (add_ass && !fmt_past_msg.empty() && fmt_past_msg.back() == '\n') {683 ss << "\n";684 };685 // format chat with new_msg686 inputs.messages.push_back(new_msg);687 inputs.add_generation_prompt = add_ass;688 auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt;689 // get the diff part690 ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());691 return ss.str();692}693 694std::string common_chat_format_example(const struct common_chat_templates * tmpls,695 bool use_jinja,696 const std::map<std::string, std::string> & chat_template_kwargs) {697 common_chat_templates_inputs inputs;698 inputs.use_jinja = use_jinja;699 inputs.add_bos = tmpls->add_bos;700 inputs.add_eos = tmpls->add_eos;701 inputs.chat_template_kwargs = chat_template_kwargs;702 auto add_simple_msg = [&](auto role, auto content) {703 common_chat_msg msg;704 msg.role = role;705 msg.content = content;706 inputs.messages.push_back(msg);707 };708 add_simple_msg("system", "You are a helpful assistant");709 add_simple_msg("user", "Hello");710 add_simple_msg("assistant", "Hi there");711 add_simple_msg("user", "How are you?");712 return common_chat_templates_apply(tmpls, inputs).prompt;713}714 715#define CHATML_TEMPLATE_SRC \716 "{%- for message in messages -%}\n" \717 " {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' -}}\n" \718 "{%- endfor -%}\n" \719 "{%- if add_generation_prompt -%}\n" \720 " {{- '<|im_start|>assistant\n' -}}\n" \721 "{%- endif -%}"722 723void common_chat_templates_free(struct common_chat_templates * tmpls) {724 delete tmpls;725}726 727bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls) {728 return tmpls->has_explicit_template;729}730 731common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {732 common_chat_prompt_preset asr_preset;733 asr_preset.system = "";734 asr_preset.user = "Transcribe audio to text";735 736 if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {737 asr_preset.system = "Perform ASR.";738 asr_preset.user = "";739 }740 741 return asr_preset;742}743 744std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {745 if (!variant.empty()) {746 if (variant == "tool_use") {747 if (tmpls->template_tool_use) {748 return tmpls->template_tool_use->source();749 }750 return "";751 }752 LOG_DBG("%s: unknown template variant: %s\n", __func__, variant.c_str());753 }754 return tmpls->template_default->source();755}756 757common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,758 const std::string & chat_template_override,759 const std::string & bos_token_override,760 const std::string & eos_token_override) {761 std::string default_template_src;762 std::string template_tool_use_src;763 764 bool has_explicit_template = !chat_template_override.empty();765 if (chat_template_override.empty()) {766 GGML_ASSERT(model != nullptr);767 const auto * str = llama_model_chat_template(model, /* name */ nullptr);768 if (str) {769 default_template_src = str;770 has_explicit_template = true;771 }772 str = llama_model_chat_template(model, /* name */ "tool_use");773 if (str) {774 template_tool_use_src = str;775 has_explicit_template = true;776 }777 } else {778 default_template_src = chat_template_override;779 }780 if (default_template_src.empty() || default_template_src == "chatml") {781 if (!template_tool_use_src.empty()) {782 default_template_src = template_tool_use_src;783 } else {784 default_template_src = CHATML_TEMPLATE_SRC;785 }786 }787 788 // TODO @ngxson : this is a temporary hack to prevent chat template from throwing an error789 // Ref: https://github.com/ggml-org/llama.cpp/pull/15230#issuecomment-3173959633790 if (default_template_src.find("<|channel|>") != std::string::npos791 // search for the error message and patch it792 && default_template_src.find("in message.content or") != std::string::npos) {793 string_replace_all(default_template_src,794 "{%- if \"<|channel|>analysis<|message|>\" in message.content or "795 "\"<|channel|>final<|message|>\" in message.content %}",796 "{%- if false %}");797 }798 799 // TODO @aldehir : this is a temporary fix, pending Minja changes800 // Ref: https://github.com/ggml-org/llama.cpp/pull/17713#issuecomment-3631342664801 if (default_template_src.find("[TOOL_CALLS]") != std::string::npos802 // search for the error message and patch it803 && default_template_src.find("if (message['content'] is none or") != std::string::npos) {804 string_replace_all(default_template_src,805 "{%- if (message['content'] is none or message['content'] == '' or "806 "message['content']|length == 0) and (message['tool_calls'] is not defined or "807 "message['tool_calls'] is none or message['tool_calls']|length == 0) %}",808 "{%- if false %}");809 }810 811 std::string token_bos = bos_token_override;812 std::string token_eos = eos_token_override;813 bool add_bos = false;814 bool add_eos = false;815 if (model) {816 const auto * vocab = llama_model_get_vocab(model);817 const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) {818 if (token == LLAMA_TOKEN_NULL) {819 if (default_template_src.find(jinja_variable_name) != std::string::npos ||820 template_tool_use_src.find(jinja_variable_name) != std::string::npos) {821 LOG_WRN(822 "common_chat_templates_init: warning: vocab does not have a %s token, jinja template won't "823 "work as intended.\n",824 name);825 }826 return std::string();827 }828 return common_token_to_piece(vocab, token, true);829 };830 token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");831 token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");832 add_bos = llama_vocab_get_add_bos(vocab);833 add_eos = llama_vocab_get_add_eos(vocab);834 }835 common_chat_templates_ptr tmpls(new common_chat_templates());836 tmpls->has_explicit_template = has_explicit_template;837 tmpls->add_bos = add_bos;838 tmpls->add_eos = add_eos;839 try {840 tmpls->template_default = std::make_unique<common_chat_template>(default_template_src, token_bos, token_eos);841 } catch (const std::exception & e) {842 LOG_ERR("%s: error: %s\n", __func__, e.what());843 LOG_ERR("%s: failed to initialize chat template\n", __func__);844 LOG_ERR("%s: please consider disabling jinja via --no-jinja, or using another chat template\n", __func__);845 throw e;846 }847 if (!template_tool_use_src.empty()) {848 try {849 tmpls->template_tool_use = std::make_unique<common_chat_template>(template_tool_use_src, token_bos, token_eos);850 } catch (const std::exception & e) {851 LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());852 }853 }854 return tmpls;855}856 857const char * common_chat_format_name(common_chat_format format) {858 switch (format) {859 case COMMON_CHAT_FORMAT_CONTENT_ONLY:860 return "Content-only";861 case COMMON_CHAT_FORMAT_PEG_SIMPLE:862 return "peg-simple";863 case COMMON_CHAT_FORMAT_PEG_NATIVE:864 return "peg-native";865 case COMMON_CHAT_FORMAT_PEG_GEMMA4:866 return "peg-gemma4";867 case COMMON_CHAT_FORMAT_PEG_MINIMAX_M3:868 return "peg-minimax-m3";869 default:870 throw std::runtime_error("Unknown chat format");871 }872}873 874const char * common_reasoning_format_name(common_reasoning_format format) {875 switch (format) {876 case COMMON_REASONING_FORMAT_NONE:877 return "none";878 case COMMON_REASONING_FORMAT_AUTO:879 return "auto";880 case COMMON_REASONING_FORMAT_DEEPSEEK:881 return "deepseek";882 case COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY:883 return "deepseek-legacy";884 default:885 throw std::runtime_error("Unknown reasoning format");886 }887}888 889common_reasoning_format common_reasoning_format_from_name(const std::string & format) {890 if (format == "none") {891 return COMMON_REASONING_FORMAT_NONE;892 }893 if (format == "auto") {894 return COMMON_REASONING_FORMAT_AUTO;895 }896 if (format == "deepseek") {897 return COMMON_REASONING_FORMAT_DEEPSEEK;898 }899 if (format == "deepseek-legacy") {900 return COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY;901 }902 throw std::runtime_error("Unknown reasoning format: " + format);903}904 905std::string common_chat_template_direct_apply_impl(906 const common_chat_template & tmpl,907 const autoparser::generation_params & inputs,908 const std::optional<json> & messages_override,909 const std::optional<json> & tools_override,910 const std::optional<json> & additional_context) {911 jinja::context ctx(tmpl.source());912 913 // messages_override is already built for this template, do not touch its content parts914 json inp = json{915 {"messages", messages_override.has_value()916 ? *messages_override917 : messages_inp_normalizer(tmpl.original_caps()).normalize(inputs.messages)},918 {"bos_token", tmpl.bos_token()},919 {"eos_token", tmpl.eos_token()},920 {"enable_thinking", inputs.enable_thinking},921 };922 if (tools_override.has_value() || !inputs.tools.empty()) {923 inp["tools"] = tools_override.has_value() ? *tools_override : inputs.tools;924 }925 if (inputs.extra_context.is_object()) {926 // TODO: do we need to merge, or replacing is fine?927 for (const auto & [k, v] : inputs.extra_context.items()) {928 inp[k] = v;929 }930 }931 if (additional_context.has_value()) {932 // TODO: merge properly instead of overwriting (matching old behavior)933 for (const auto & [k, v] : additional_context->items()) {934 inp[k] = v;935 }936 }937 if (inputs.add_generation_prompt) {938 inp["add_generation_prompt"] = true;939 }940 if (inp.contains("preserve_reasoning") && inp["preserve_reasoning"].is_boolean()) {941 bool enabled = inp["preserve_reasoning"].get<bool>();942 jinja::caps_apply_preserve_reasoning(ctx, enabled);943 }944 if (inp.contains("reasoning_effort") && inp["reasoning_effort"].is_string() && !inp["reasoning_effort"].empty()) {945 std::string reasoning_effort = inp["reasoning_effort"].get<std::string>();946 jinja::caps_apply_reasoning_effort(ctx, reasoning_effort);947 }948 949 jinja::global_from_json(ctx, inp, inputs.mark_input);950 951 // render952 jinja::runtime runtime(ctx);953 const jinja::value results = runtime.execute(tmpl.prog);954 auto parts = jinja::runtime::gather_string_parts(results);955 956 std::string result = parts->as_string().str();957 958 // TODO: improve this later959 if (inputs.add_bos && string_starts_with(result, tmpl.bos_token())) {960 result = result.substr(tmpl.bos_token().size());961 }962 if (inputs.add_eos && string_ends_with(result, tmpl.eos_token())) {963 result = result.substr(0, result.size() - tmpl.eos_token().size());964 }965 return result;966}967 968std::string common_chat_template_direct_apply(969 const common_chat_template & tmpl,970 const autoparser::generation_params & inputs) {971 return common_chat_template_direct_apply_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);972}973 974std::string common_chat_template_generation_prompt_impl(975 const common_chat_template & tmpl,976 const autoparser::generation_params & inputs,977 const std::optional<json> & messages_override,978 const std::optional<json> & tools_override,979 const std::optional<json> & additional_context) {980 981 autoparser::generation_params params = inputs;982 params.add_generation_prompt = false;983 params.continue_final_message = COMMON_CHAT_CONTINUATION_NONE;984 std::string no_gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context);985 params.add_generation_prompt = true;986 std::string gen_prompt = common_chat_template_direct_apply_impl(tmpl, params, messages_override, tools_override, additional_context);987 988 size_t prefix_len = 0;989 size_t min_size = std::min(no_gen_prompt.size(), gen_prompt.size());990 while (prefix_len < min_size && no_gen_prompt[prefix_len] == gen_prompt[prefix_len]) {991 prefix_len++;992 }993 return gen_prompt.substr(prefix_len);994}995 996std::string common_chat_template_generation_prompt(997 const common_chat_template & tmpl,998 const autoparser::generation_params & inputs) {999 return common_chat_template_generation_prompt_impl(tmpl, inputs, std::nullopt, std::nullopt, std::nullopt);1000}1001 1002namespace workaround {1003 1004static void map_developer_role_to_system(json & messages) {1005 for (auto & message : messages) {1006 if (message.contains("role")) {1007 if (message["role"] == "developer") {1008 message["role"] = "system";1009 }1010 }1011 }1012}1013 1014 1015// if first message is system and template does not support it, merge it with next message1016static void system_message_not_supported(json & messages) {1017 if (!messages.empty() && messages.front().at("role") == "system") {1018 if (messages.size() > 1) {1019 LOG_DBG("Merging system prompt into next message\n");1020 auto & first_msg = messages.front();1021 auto & second_msg = messages[1];1022 second_msg["content"] = first_msg.at("content").get<std::string>()1023 + "\n" + second_msg.at("content").get<std::string>();1024 messages.erase(0);1025 } else {1026 LOG_WRN("Removing system prompt due to template not supporting system role\n");1027 messages.erase(0);1028 }1029 }1030}1031 1032static void requires_non_null_content(json & messages) {1033 GGML_ASSERT(messages.is_array());1034 for (auto & message : messages) {1035 if (message.contains("tool_calls") && !message.contains("content")) {1036 message["content"] = "";1037 }1038 }1039}1040 1041static void func_args_not_string(json & messages) {1042 GGML_ASSERT(messages.is_array());1043 for (auto & message : messages) {1044 if (message.contains("tool_calls")) {1045 for (auto & tool_call : message["tool_calls"]) {1046 if (tool_call.contains("function") && tool_call["function"].contains("arguments")) {1047 auto & args = tool_call["function"]["arguments"];1048 if (args.is_string()) {1049 try {1050 args = json::parse(args.get<std::string>());1051 } catch (const std::exception & e) {1052 throw std::runtime_error("Failed to parse tool call arguments as JSON: " + std::string(e.what()));1053 }1054 }1055 }1056 }1057 }1058 }1059}1060 1061// Trim leading/trailing whitespace from message contents before rendering. This1062// has to run on the messages (not on the rendered JSON) because templates with1063// string-only content caps concatenate typed content parts into a single string1064// during rendering, after which the per-part whitespace can no longer be reached.1065// Both the plain string content and the text of typed content parts are trimmed.1066static void trim_all_content(std::vector<common_chat_msg> & messages) {1067 for (auto & message : messages) {1068 message.content = trim_whitespace(message.content);1069 message.reasoning_content = trim_whitespace(message.reasoning_content);1070 for (auto & part : message.content_parts) {1071 if (part.type == "text") {1072 part.text = trim_whitespace(part.text);1073 }1074 }1075 }1076}1077 1078}1079 1080static json common_chat_extra_context() {1081 json ctx = json::object();1082 std::chrono::system_clock::time_point now = std::chrono::system_clock::now();1083 std::string datetime_str = format_time(now, "%b %d %Y");1084 std::string date_str = format_time(now, "%d %b %Y");1085 ctx["datetime"] = datetime_str;1086 ctx["date_string"] = date_str;1087 return ctx;1088}1089 1090std::optional<common_chat_params> common_chat_try_specialized_template(1091 const common_chat_template & tmpl,1092 const std::string & src,1093 autoparser::generation_params & params) {1094 // Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser1095 // Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them1096 if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos &&1097 src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) {1098 LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n");1099 return common_chat_params_init_ministral_3(tmpl, params);1100 }1101 1102 // GPT-OSS - has unique channel-based structure that needs dedicated handler1103 if (src.find("<|channel|>") != std::string::npos) {1104 LOG_DBG("Using specialized template: GPT-OSS\n");1105 return common_chat_params_init_gpt_oss(tmpl, params);1106 }1107 1108 // Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.1109 if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {1110 LOG_DBG("Using specialized template: Muse Glimmer\n");1111 return common_chat_params_init_muse_glimmer(tmpl, params);1112 }1113 1114 // Functionary v3.2 - uses recipient-based format with >>>recipient\n{content}1115 // Detection: template has ">>>all" for content and ">>>" prefix for tool calls1116 if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) {1117 LOG_DBG("Using specialized template: Functionary v3.2\n");1118 return common_chat_params_init_functionary_v3_2(tmpl, params);1119 }1120 1121 // Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>1122 // Detection: template has "<|tool_calls_section_begin|>" and "functions." prefix in tool call IDs1123 if (src.find("<|tool_calls_section_begin|>") != std::string::npos &&1124 src.find("<|tool_call_begin|>") != std::string::npos) {1125 LOG_DBG("Using specialized template: Kimi K2 Thinking\n");1126 return common_chat_params_init_kimi_k2(tmpl, params);1127 }1128 1129 // Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it1130 if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos &&1131 src.find("<|end_of_msg|>") != std::string::npos) {1132 LOG_DBG("Using specialized template: Kimi K3\n");1133 return common_chat_params_init_kimi_k3(tmpl, params);1134 }1135 1136 // Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and1137 // <|START_ACTION|> JSON tool calls. <|START_TEXT|> is unique to this template (the older1138 // Command-R templates use <|START_RESPONSE|>).1139 if (src.find("<|START_TEXT|>") != std::string::npos &&1140 src.find("<|START_ACTION|>") != std::string::npos) {1141 LOG_DBG("Using specialized template: Cohere2 MoE\n");1142 return common_chat_params_init_cohere2moe(tmpl, params);1143 }1144 1145 if (is_lfm2_template(src)) {1146 LOG_DBG("Using specialized template: LFM2\n");1147 return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);1148 }1149 1150 // LFM2.5 format detection: template uses plain "List of tools: [...]" with no special tokens1151 if (src.find("List of tools: [") != std::string::npos &&1152 src.find("<|tool_list_start|>") == std::string::npos) {1153 LOG_DBG("Using specialized template: LFM2.5\n");1154 return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ false);1155 }1156 1157 // GigaChatV3 format detection1158 if (src.find("<|role_sep|>") != std::string::npos &&1159 src.find("<|message_sep|>") != std::string::npos &&1160 src.find("<|function_call|>") == std::string::npos) {1161 LOG_DBG("Using specialized template: GigaChatV3\n");1162 return common_chat_params_init_gigachat_v3(tmpl, params);1163 }1164 1165 // MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's1166 // markup delimiters, so detect the template and use a dedicated parser.1167 if (src.find("]<]minimax[>[") != std::string::npos &&1168 src.find("<tool_call>") != std::string::npos &&1169 src.find("<invoke name=") != std::string::npos) {1170 LOG_DBG("Using specialized template: MiniMax-M3\n");1171 return common_chat_params_init_minimax_m3(tmpl, params);1172 }1173 1174 // DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.1175 // The template source contains the token as a variable assignment, not as a literal in markup.1176 // V3.2 names the tool call block "function_calls", V4 names it "tool_calls".1177 if (src.find("dsml_token") != std::string::npos &&1178 src.find("DSML") != std::string::npos &&1179 (src.find("function_calls") != std::string::npos ||1180 src.find("tool_calls") != std::string::npos)) {1181 LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");1182 return common_chat_params_init_deepseek_v3_2(tmpl, params);1183 }1184 1185 // Gemma4 format detection1186 if (src.find("'<|tool_call>call:'") != std::string::npos) {1187 if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {1188 // apply workarounds if using the older gemma4 templates1189 LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "1190 "Consider updating to the official template.\n", __func__);1191 workaround::convert_tool_responses_gemma4(params.messages);1192 }1193 return common_chat_params_init_gemma4(tmpl, params);1194 }1195 1196 // MiniCPM5 - XML tool calls with <function name="..."><param name="...">...</param></function>1197 if (src.find("Tool usage guidelines:") != std::string::npos &&1198 src.find("<function name=\"") != std::string::npos &&1199 src.find("<param name=\"") != std::string::npos) {1200 LOG_DBG("Using specialized template: MiniCPM5\n");