Codeprocastinator/optimized-tinyllama-covalent
0119
1#include "chat.h"2#include "json-schema-to-grammar.h"3#include "log.h"4#include "minja/chat-template.hpp"5#include "minja/minja.hpp"6 7#include <optional>8 9typedef minja::chat_template common_chat_template;10 11struct common_chat_templates {12 bool has_explicit_template; // Model had builtin template or template overridde was specified.13 std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)14 std::unique_ptr<common_chat_template> template_tool_use;15};16 17struct templates_params {18 json messages;19 json tools;20 common_chat_tool_choice tool_choice;21 json json_schema;22 bool parallel_tool_calls;23 bool stream;24 std::string grammar;25 bool add_generation_prompt = true;26 bool extract_reasoning = true;27};28 29common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {30 if (tool_choice == "auto") {31 return COMMON_CHAT_TOOL_CHOICE_AUTO;32 }33 if (tool_choice == "none") {34 return COMMON_CHAT_TOOL_CHOICE_NONE;35 }36 if (tool_choice == "required") {37 return COMMON_CHAT_TOOL_CHOICE_REQUIRED;38 }39 throw std::runtime_error("Invalid tool_choice: " + tool_choice);40}41 42template <>43std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messages) {44 std::vector<common_chat_msg> msgs;45 46 try {47 48 if (!messages.is_array()) {49 throw std::runtime_error("Expected 'messages' to be an array, got " + messages.dump());50 }51 52 for (const auto & message : messages) {53 if (!message.is_object()) {54 throw std::runtime_error("Expected 'message' to be an object, got " + message.dump());55 }56 57 common_chat_msg msg;58 if (!message.contains("role")) {59 throw std::runtime_error("Missing 'role' in message: " + message.dump());60 }61 msg.role = message.at("role");62 63 auto has_content = message.contains("content");64 auto has_tool_calls = message.contains("tool_calls");65 if (has_content) {66 const auto & content = message.at("content");67 if (content.is_string()) {68 msg.content = content;69 } else if (content.is_array()) {70 for (const auto & part : content) {71 if (!part.contains("type")) {72 throw std::runtime_error("Missing content part type: " + part.dump());73 }74 const auto & type = part.at("type");75 if (type != "text") {76 throw std::runtime_error("Unsupported content part type: " + type.dump());77 }78 common_chat_msg_content_part msg_part;79 msg_part.type = type;80 msg_part.text = part.at("text");81 msg.content_parts.push_back(msg_part);82 }83 } else if (!content.is_null()) {84 throw std::runtime_error("Invalid 'content' type: expected string or array, got " + content.dump() + " (ref: https://github.com/ggml-org/llama.cpp/issues/8367)");85 }86 }87 if (has_tool_calls) {88 for (const auto & tool_call : message.at("tool_calls")) {89 common_chat_tool_call tc;90 if (!tool_call.contains("type")) {91 throw std::runtime_error("Missing tool call type: " + tool_call.dump());92 }93 const auto & type = tool_call.at("type");94 if (type != "function") {95 throw std::runtime_error("Unsupported tool call type: " + tool_call.dump());96 }97 if (!tool_call.contains("function")) {98 throw std::runtime_error("Missing tool call function: " + tool_call.dump());99 }100 const auto & fc = tool_call.at("function");101 if (!fc.contains("name")) {102 throw std::runtime_error("Missing tool call name: " + tool_call.dump());103 }104 tc.name = fc.at("name");105 tc.arguments = fc.at("arguments");106 if (tool_call.contains("id")) {107 tc.id = tool_call.at("id");108 }109 msg.tool_calls.push_back(tc);110 }111 }112 if (!has_content && !has_tool_calls) {113 throw std::runtime_error("Expected 'content' or 'tool_calls' (ref: https://github.com/ggml-org/llama.cpp/issues/8367 & https://github.com/ggml-org/llama.cpp/issues/12279)");114 }115 if (message.contains("reasoning_content")) {116 msg.reasoning_content = message.at("reasoning_content");117 }118 if (message.contains("name")) {119 msg.tool_name = message.at("name");120 }121 if (message.contains("tool_call_id")) {122 msg.tool_call_id = message.at("tool_call_id");123 }124 125 msgs.push_back(msg);126 }127 } catch (const std::exception & e) {128 throw std::runtime_error("Failed to parse messages: " + std::string(e.what()) + "; messages = " + messages.dump(2));129 }130 131 return msgs;132}133 134template <>135json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {136 json messages = json::array();137 for (const auto & msg : msgs) {138 if (!msg.content.empty() && !msg.content_parts.empty()) {139 throw std::runtime_error("Cannot specify both content and content_parts");140 }141 json jmsg {142 {"role", msg.role},143 };144 if (!msg.content.empty()) {145 jmsg["content"] = msg.content;146 } else if (!msg.content_parts.empty()) {147 if (concat_typed_text) {148 std::string text;149 for (const auto & part : msg.content_parts) {150 if (part.type != "text") {151 LOG_WRN("Ignoring content part type: %s\n", part.type.c_str());152 continue;153 }154 if (!text.empty()) {155 text += '\n';156 }157 text += part.text;158 }159 jmsg["content"] = text;160 } else {161 auto & parts = jmsg["content"] = json::array();162 for (const auto & part : msg.content_parts) {163 parts.push_back({164 {"type", part.type},165 {"text", part.text},166 });167 }168 }169 } else {170 jmsg["content"] = json(); // null171 }172 if (!msg.reasoning_content.empty()) {173 jmsg["reasoning_content"] = msg.reasoning_content;174 }175 if (!msg.tool_name.empty()) {176 jmsg["name"] = msg.tool_name;177 }178 if (!msg.tool_call_id.empty()) {179 jmsg["tool_call_id"] = msg.tool_call_id;180 }181 if (!msg.tool_calls.empty()) {182 auto & tool_calls = jmsg["tool_calls"] = json::array();183 for (const auto & tool_call : msg.tool_calls) {184 json tc {185 {"type", "function"},186 {"function", {187 {"name", tool_call.name},188 {"arguments", tool_call.arguments},189 }},190 };191 if (!tool_call.id.empty()) {192 tc["id"] = tool_call.id;193 }194 tool_calls.push_back(tc);195 }196 }197 messages.push_back(jmsg);198 }199 return messages;200}201 202template <>203std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const std::string & messages) {204 return common_chat_msgs_parse_oaicompat(json::parse(messages));205}206 207template <>208std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & tools) {209 std::vector<common_chat_tool> result;210 211 try {212 if (!tools.is_null()) {213 if (!tools.is_array()) {214 throw std::runtime_error("Expected 'tools' to be an array, got " + tools.dump());215 }216 for (const auto & tool : tools) {217 if (!tool.contains("type")) {218 throw std::runtime_error("Missing tool type: " + tool.dump());219 }220 const auto & type = tool.at("type");221 if (!type.is_string() || type != "function") {222 throw std::runtime_error("Unsupported tool type: " + tool.dump());223 }224 if (!tool.contains("function")) {225 throw std::runtime_error("Missing tool function: " + tool.dump());226 }227 228 const auto & function = tool.at("function");229 result.push_back({230 /* .name = */ function.at("name"),231 /* .description = */ function.at("description"),232 /* .parameters = */ function.at("parameters").dump(),233 });234 }235 }236 } catch (const std::exception & e) {237 throw std::runtime_error("Failed to parse tools: " + std::string(e.what()) + "; tools = " + tools.dump(2));238 }239 240 return result;241}242 243template <>244std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const std::string & tools) {245 return common_chat_tools_parse_oaicompat(json::parse(tools));246}247 248template <>249json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools) {250 if (tools.empty()) {251 return json();252 }253 254 auto result = json::array();255 for (const auto & tool : tools) {256 result.push_back({257 {"type", "function"},258 {"function", {259 {"name", tool.name},260 {"description", tool.description},261 {"parameters", json::parse(tool.parameters)},262 }},263 });264 }265 return result;266}267 268bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {269 if (use_jinja) {270 try {271 common_chat_msg msg;272 msg.role = "user";273 msg.content = "test";274 275 auto tmpls = common_chat_templates_init(/* model= */ nullptr, tmpl);276 277 common_chat_templates_inputs inputs;278 inputs.messages = {msg};279 280 common_chat_templates_apply(tmpls.get(), inputs);281 return true;282 } catch (const std::exception & e) {283 LOG_ERR("%s: failed to apply template: %s\n", __func__, e.what());284 return false;285 }286 }287 llama_chat_message chat[] = {{"user", "test"}};288 const int res = llama_chat_apply_template(tmpl.c_str(), chat, 1, true, nullptr, 0);289 return res >= 0;290}291 292std::string common_chat_format_single(293 const struct common_chat_templates * tmpls,294 const std::vector<common_chat_msg> & past_msg,295 const common_chat_msg & new_msg,296 bool add_ass,297 bool use_jinja) {298 299 common_chat_templates_inputs inputs;300 inputs.use_jinja = use_jinja;301 302 std::string fmt_past_msg;303 if (!past_msg.empty()) {304 inputs.messages = past_msg;305 inputs.add_generation_prompt = false;306 fmt_past_msg = common_chat_templates_apply(tmpls, inputs).prompt;307 }308 std::ostringstream ss;309 // if the past_msg ends with a newline, we must preserve it in the formatted version310 if (add_ass && !fmt_past_msg.empty() && fmt_past_msg.back() == '\n') {311 ss << "\n";312 };313 // format chat with new_msg314 inputs.messages.push_back(new_msg);315 inputs.add_generation_prompt = add_ass;316 auto fmt_new_msg = common_chat_templates_apply(tmpls, inputs).prompt;317 // get the diff part318 ss << fmt_new_msg.substr(fmt_past_msg.size(), fmt_new_msg.size() - fmt_past_msg.size());319 return ss.str();320}321 322std::string common_chat_format_example(const struct common_chat_templates * tmpls, bool use_jinja) {323 common_chat_templates_inputs inputs;324 inputs.use_jinja = use_jinja;325 auto add_simple_msg = [&](auto role, auto content) {326 common_chat_msg msg;327 msg.role = role;328 msg.content = content;329 inputs.messages.push_back(msg);330 };331 add_simple_msg("system", "You are a helpful assistant");332 add_simple_msg("user", "Hello");333 add_simple_msg("assistant", "Hi there");334 add_simple_msg("user", "How are you?");335 return common_chat_templates_apply(tmpls, inputs).prompt;336}337 338#define CHATML_TEMPLATE_SRC \339 "{%- for message in messages -%}\n" \340 " {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' -}}\n" \341 "{%- endfor -%}\n" \342 "{%- if add_generation_prompt -%}\n" \343 " {{- '<|im_start|>assistant\n' -}}\n" \344 "{%- endif -%}"345 346void common_chat_templates_free(struct common_chat_templates * tmpls) {347 delete tmpls;348}349 350bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls) {351 return tmpls->has_explicit_template;352}353 354const char * common_chat_templates_source(const struct common_chat_templates * tmpls, const char * variant) {355 if (variant != nullptr) {356 if (strcmp(variant, "tool_use") == 0) {357 if (tmpls->template_tool_use) {358 return tmpls->template_tool_use->source().c_str();359 }360 return nullptr;361 } else {362 LOG_DBG("%s: unknown template variant: %s\n", __func__, variant);363 }364 }365 return tmpls->template_default->source().c_str();366}367 368common_chat_templates_ptr common_chat_templates_init(369 const struct llama_model * model,370 const std::string & chat_template_override,371 const std::string & bos_token_override,372 const std::string & eos_token_override)373{374 std::string default_template_src;375 std::string template_tool_use_src;376 377 bool has_explicit_template = !chat_template_override.empty();378 if (chat_template_override.empty()) {379 GGML_ASSERT(model != nullptr);380 const auto * str = llama_model_chat_template(model, /* name */ nullptr);381 if (str) {382 default_template_src = str;383 has_explicit_template = true;384 }385 str = llama_model_chat_template(model, /* name */ "tool_use");386 if (str) {387 template_tool_use_src = str;388 has_explicit_template = true;389 }390 } else {391 default_template_src = chat_template_override;392 }393 if (default_template_src.empty() || default_template_src == "chatml") {394 if (!template_tool_use_src.empty()) {395 default_template_src = template_tool_use_src;396 } else {397 default_template_src = CHATML_TEMPLATE_SRC;398 }399 }400 std::string token_bos = bos_token_override;401 std::string token_eos = eos_token_override;402 if (model) {403 const auto * vocab = llama_model_get_vocab(model);404 const auto get_token = [&](llama_token token, const char * name, const char * jinja_variable_name) {405 if (token == LLAMA_TOKEN_NULL) {406 if (default_template_src.find(jinja_variable_name) != std::string::npos407 || template_tool_use_src.find(jinja_variable_name) != std::string::npos) {408 LOG_WRN("common_chat_templates_init: warning: vocab does not have a %s token, jinja template won't work as intended.\n", name);409 }410 return std::string();411 }412 return common_token_to_piece(vocab, token, true);413 };414 token_bos = get_token(llama_vocab_bos(vocab), "BOS", "bos_token");415 token_eos = get_token(llama_vocab_eos(vocab), "EOS", "eos_token");416 }417 common_chat_templates_ptr tmpls(new common_chat_templates());418 tmpls->has_explicit_template = has_explicit_template;419 try {420 tmpls->template_default = std::make_unique<minja::chat_template>(default_template_src, token_bos, token_eos);421 } catch (const std::exception & e) {422 LOG_ERR("%s: failed to parse chat template (defaulting to chatml): %s \n", __func__, e.what());423 tmpls->template_default = std::make_unique<minja::chat_template>(CHATML_TEMPLATE_SRC, token_bos, token_eos);424 }425 if (!template_tool_use_src.empty()) {426 try {427 tmpls->template_tool_use = std::make_unique<minja::chat_template>(template_tool_use_src, token_bos, token_eos);428 } catch (const std::exception & e) {429 LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());430 }431 }432 return tmpls;433}434 435std::string common_chat_format_name(common_chat_format format) {436 switch (format) {437 case COMMON_CHAT_FORMAT_CONTENT_ONLY: return "Content-only";438 case COMMON_CHAT_FORMAT_GENERIC: return "Generic";439 case COMMON_CHAT_FORMAT_MISTRAL_NEMO: return "Mistral Nemo";440 case COMMON_CHAT_FORMAT_LLAMA_3_X: return "Llama 3.x";441 case COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS: return "Llama 3.x with builtin tools";442 case COMMON_CHAT_FORMAT_DEEPSEEK_R1: return "DeepSeek R1";443 case COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING: return "DeepSeek R1 (extract reasoning)";444 case COMMON_CHAT_FORMAT_FIREFUNCTION_V2: return "FireFunction v2";445 case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2: return "Functionary v3.2";446 case COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1: return "Functionary v3.1 Llama 3.1";447 case COMMON_CHAT_FORMAT_HERMES_2_PRO: return "Hermes 2 Pro";448 case COMMON_CHAT_FORMAT_HERMES_2_PRO_EXTRACT_REASONING: return "Hermes 2 Pro (extract reasoning)";449 case COMMON_CHAT_FORMAT_COMMAND_R7B: return "Command R7B";450 case COMMON_CHAT_FORMAT_COMMAND_R7B_EXTRACT_REASONING: return "Command R7B (extract reasoning)";451 default:452 throw std::runtime_error("Unknown chat format");453 }454}455 456static bool parse_json(std::string::const_iterator & it, const std::string::const_iterator & end, json & out) {457 // // https://json.nlohmann.me/features/parsing/sax_interface/458 struct json_error_locator : public nlohmann::json_sax<json> {459 std::size_t position;460 bool found_error;461 462 json_error_locator() : position(0), found_error(false) {}463 464 bool parse_error(std::size_t position, const std::string &, const json::exception &) override { // NOLINT465 this->position = position - 1;466 this->found_error = true;467 return false;468 }469 bool null() override { return true; } // NOLINT470 bool boolean(bool) override { return true; } // NOLINT471 bool number_integer(number_integer_t) override { return true; } // NOLINT472 bool number_unsigned(number_unsigned_t) override { return true; } // NOLINT473 bool number_float(number_float_t, const string_t &) override { return true; } // NOLINT474 bool string(string_t &) override { return true; } // NOLINT475 bool binary(binary_t &) override { return true; } // NOLINT476 bool start_object(std::size_t) override { return true; } // NOLINT477 bool key(string_t &) override { return true; } // NOLINT478 bool end_object() override { return true; }479 bool start_array(std::size_t) override { return true; } // NOLINT480 bool end_array() override { return true; }481 };482 json_error_locator err_loc;483 json::sax_parse(it, end, &err_loc);484 485 std::string::const_iterator temptative_end;486 if (err_loc.found_error) {487 temptative_end = it + err_loc.position;488 } else {489 temptative_end = end;490 }491 std::string json_sub {it, temptative_end};492 try {493 out = json::parse(json_sub);494 it = temptative_end;495 return true;496 } catch (const std::exception &) {497 return false;498 }499}500 501static bool parse_literal(std::string::const_iterator & it, const std::string::const_iterator & end, const std::string & expected) {502 auto expected_it = expected.begin();503 auto tmp_it = it;504 while (tmp_it != end && expected_it != expected.end() && *tmp_it == *expected_it) {505 ++tmp_it;506 ++expected_it;507 }508 if (expected_it == expected.end()) {509 it = tmp_it;510 return true;511 }512 return false;513}514 515static std::optional<std::smatch> parse_pattern(std::string::const_iterator & it, const std::string::const_iterator & end, const std::regex & expected) {516 std::smatch match;517 if (std::regex_match(it, end, match, expected)) {518 it = match.suffix().first;519 return match;520 }521 return std::nullopt;522}523 524static void consume_spaces(std::string::const_iterator & it, const std::string::const_iterator & end) {525 while (it != end && std::isspace(*it)) {526 ++it;527 }528}529 530/**531 * Takes a prefix regex that must have 1 group to capture the function name, a closing suffix, and expects json parameters in between.532 * Aggregates the prefix, suffix and in-between text into the content.533 */534static common_chat_msg parse_json_tool_calls(535 const std::string& input,536 const std::optional<std::regex> & trigger_opt,537 const std::regex & function_regex,538 const std::regex & close_regex,539 bool allow_raw_python = false) {540 std::smatch match;541 542 common_chat_msg result;543 result.role = "assistant";544 545 546 auto end = input.end();547 auto it = input.begin();548 549 if (trigger_opt) {550 if (!std::regex_search(it, end, match, *trigger_opt)) {551 result.content = input;552 return result;553 }554 result.content = match.prefix().str();555 it = match.suffix().first;556 }557 558 while (it != end) {559 std::sregex_iterator rend;560 std::sregex_iterator rit(it, end, function_regex);561 if (rit == rend) {562 result.content += std::string(it, end);563 break;564 }565 auto name = rit->str(1);566 result.content += std::string(it, rit->prefix().second);567 it = rit->suffix().first;568 569 json arguments;570 if (parse_json(it, end, arguments)) {571 if (!std::regex_search(it, end, match, close_regex)) {572 throw std::runtime_error("Malformed input, missing closing pattern: " + input);573 }574 it = match.suffix().first;575 result.tool_calls.push_back({name, arguments.is_string() ? arguments.get<std::string>() : arguments.dump(), /* id= */ ""});576 } else {577 if (allow_raw_python && name == "python") {578 result.tool_calls.push_back({name, json({{"code", std::string(it, end)}}).dump(), /* id= */ ""});579 break;580 }581 throw std::runtime_error("Failed to parse json tool call arguments: " + input);582 }583 }584 585 if (!result.tool_calls.empty()) {586 if (!string_strip(result.content).empty()) {587 LOG_WRN("Content found with tool calls: %s\n", result.content.c_str());588 }589 result.content = "";590 }591 return result;592}593 594static common_chat_tool_call process_tool_call(const json & tool_call) {595 const auto & arguments = tool_call.at("arguments");596 return {597 /* .name = */ tool_call.at("name"),598 /* .arguments = */ arguments.is_string() ? arguments.get<std::string>() : arguments.dump(),599 /* .id = */ tool_call.contains("id") ? tool_call.at("id") : "",600 };601}602static common_chat_msg parse_prefixed_json_tool_call_array(const std::string& input, const std::string & prefix, size_t rstrip_prefix = 0) {603 auto content_end = input.find(prefix);604 size_t tc_start = std::string::npos;605 606 common_chat_msg result;607 result.role = "assistant";608 if (content_end == std::string::npos) {609 result.content = input;610 } else {611 tc_start = content_end + prefix.size() - rstrip_prefix;612 result.content = input.substr(0, content_end);613 auto tool_calls = json::parse(input.substr(tc_start));614 for (const auto & tool_call : tool_calls) {615 result.tool_calls.emplace_back(process_tool_call(tool_call));616 }617 }618 return result;619}620 621static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {622 for (const auto & tool : tools) {623 if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {624 LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());625 continue;626 }627 fn(tool);628 }629}630 631static std::string apply(632 const common_chat_template & tmpl,633 const nlohmann::ordered_json & messages,634 const nlohmann::ordered_json & tools,635 bool add_generation_prompt,636 const nlohmann::ordered_json & extra_context = nlohmann::ordered_json())637{638 minja::chat_template_inputs tmpl_inputs;639 tmpl_inputs.messages = messages;640 tmpl_inputs.tools = tools;641 tmpl_inputs.add_generation_prompt = add_generation_prompt;642 tmpl_inputs.extra_context = extra_context;643 // TODO: add flag to control date/time, if only for testing purposes.644 // tmpl_inputs.now = std::chrono::system_clock::now();645 646 minja::chat_template_options tmpl_opts;647 // To avoid double BOS / EOS tokens, we're manually removing begining / trailing tokens648 // instead of using `chat_template_options.use_bos_token = false`, since these tokens649 // may be needed inside the template / between messages too.650 auto result = tmpl.apply(tmpl_inputs, tmpl_opts);651 if (string_starts_with(result, tmpl.bos_token())) {652 result = result.substr(tmpl.bos_token().size());653 }654 if (string_ends_with(result, tmpl.eos_token())) {655 result = result.substr(0, result.size() - tmpl.eos_token().size());656 }657 return result;658}659 660static common_chat_params common_chat_params_init_generic(const common_chat_template & tmpl, const struct templates_params & inputs) {661 common_chat_params data;662 663 auto tool_call_schemas = json::array();664 foreach_function(inputs.tools, [&](const json & tool) {665 const auto & function = tool.at("function");666 auto tool_schema = json {667 {"type", "object"},668 {"properties", {669 {"name", {670 {"type", "string"},671 {"const", function.at("name")},672 }},673 {"arguments", function.at("parameters")},674 }},675 {"required", json::array({"name", "arguments"})},676 };677 if (function.contains("description")) {678 tool_schema["description"] = function.at("description");679 }680 if (inputs.parallel_tool_calls) {681 tool_schema.at("properties")["id"] = {682 {"type", "string"},683 {"minLength", 4},684 };685 tool_schema.at("required").push_back("id");686 }687 tool_call_schemas.emplace_back(tool_schema);688 });689 const auto tool_call =690 inputs.parallel_tool_calls691 ? json {692 {"type", "object"},693 {"properties", {694 {"tool_calls", {695 {"type", "array"},696 {"items", tool_call_schemas.size() == 1 ? tool_call_schemas[0] : json {697 {"anyOf", tool_call_schemas},698 }},699 {"minItems", 1},700 }},701 }},702 {"required", json::array({"tool_calls"})},703 }704 : json {705 {"type", "object"},706 {"properties", {707 {"tool_call", tool_call_schemas.size() == 1 ? tool_call_schemas[0] : json {708 {"anyOf", tool_call_schemas},709 }},710 }},711 {"required", json::array({"tool_call"})},712 };713 const auto schema =714 inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED715 ? json {716 {"anyOf", json::array({717 tool_call,718 {719 {"type", "object"},720 {"properties", {721 {"response", inputs.json_schema.is_null()722 ? json {{"type", "string"}}723 : inputs.json_schema724 },725 }},726 {"required", json::array({"response"})},727 },728 })}729 }730 : tool_call;731 732 data.grammar_lazy = false;733 data.grammar = build_grammar([&](const common_grammar_builder & builder) {734 builder.add_schema("root", schema);735 });736 737 auto tweaked_messages = common_chat_template::add_system(738 inputs.messages,739 "Respond in JSON format, either with `tool_call` (a request to call tools) or with `response` reply to the user's request");740 741 data.prompt = apply(tmpl, tweaked_messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);742 data.format = COMMON_CHAT_FORMAT_GENERIC;743 return data;744}745static common_chat_msg common_chat_parse_generic(const std::string & input) {746 json data = json::parse(input);747 common_chat_msg result;748 result.role = "assistant";749 if (data.contains("tool_calls")) {750 for (const auto & tool_call : data.at("tool_calls")) {751 result.tool_calls.push_back({752 tool_call.at("name"),753 tool_call.at("arguments").dump(),754 tool_call.contains("id") ? tool_call.at("id") : "",755 });756 }757 } else if (data.contains("tool_call")) {758 result.tool_calls.push_back({759 data.at("tool_call").at("name"),760 data.at("tool_call").at("arguments").dump(),761 /* id= */ "",762 });763 } else if (data.contains("response")) {764 const auto & response = data.at("response");765 result.content = response.is_string() ? response.get<std::string>() : response.dump(2);766 }767 return result;768}769 770static common_chat_params common_chat_params_init_mistral_nemo(const common_chat_template & tmpl, const struct templates_params & inputs) {771 common_chat_params data;772 data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;773 data.grammar = build_grammar([&](const common_grammar_builder & builder) {774 auto schemas = json::array();775 foreach_function(inputs.tools, [&](const json & tool) {776 const auto & function = tool.at("function");777 schemas.push_back({778 {"type", "object"},779 {"properties", {780 // Important note: the model is probably trained to take a JSON stringified arguments value.781 // It's hard to constrain that for now (while reusing the JSON schema conversion), so we're just expecting a plain object.782 {"name", {783 {"type", "string"},784 {"const", function.at("name")},785 }},786 {"arguments", function.at("parameters")},787 {"id", {788 {"type", "string"},789 // Nemo's template expects a 9-character alphanumeric ID.790 {"pattern", "^[a-zA-Z0-9]{9}$"},791 }},792 }},793 {"required", json::array({"name", "arguments", "id"})},794 });795 });796 auto schema = json {797 {"type", "array"},798 {"items", schemas.size() == 1 ? schemas[0] : json {{"anyOf", schemas}}},799 {"minItems", 1},800 };801 if (!inputs.parallel_tool_calls) {802 schema["maxItems"] = 1;803 }804 builder.add_rule("root", "\"[TOOL_CALLS]\" " + builder.add_schema("tool_calls", schema));805 });806 data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]"});807 data.preserved_tokens = {808 "[TOOL_CALLS]",809 };810 data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);811 data.format = COMMON_CHAT_FORMAT_MISTRAL_NEMO;812 return data;813}814static common_chat_msg common_chat_parse_mistral_nemo(const std::string & input) {815 return parse_prefixed_json_tool_call_array(input, "[TOOL_CALLS]");816}817 818static common_chat_params common_chat_params_init_command_r7b(const common_chat_template & tmpl, const struct templates_params & inputs) {819 common_chat_params data;820 data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;821 data.grammar = build_grammar([&](const common_grammar_builder & builder) {822 auto schemas = json::array();823 foreach_function(inputs.tools, [&](const json & tool) {824 const auto & function = tool.at("function");825 schemas.push_back({826 {"type", "object"},827 {"properties", {828 {"tool_call_id", {829 {"type", "string"},830 // Command-R's template expects an integer string.831 {"pattern", "^[0-9]{1,10}$"},832 }},833 {"tool_name", {834 {"type", "string"},835 {"const", function.at("name")},836 }},837 {"parameters", function.at("parameters")},838 }},839 {"required", json::array({"tool_call_id", "tool_name", "parameters"})},840 });841 });842 auto schema = json {843 {"type", "array"},844 {"items", schemas.size() == 1 ? schemas[0] : json {{"anyOf", schemas}}},845 {"minItems", 1},846 };847 if (!inputs.parallel_tool_calls) {848 schema["maxItems"] = 1;849 }850 builder.add_rule("root", "\"<|START_ACTION|>\" " + builder.add_schema("tool_calls", schema) + " \"<|END_ACTION|>\"");851 });852 data.grammar_triggers.push_back({853 COMMON_GRAMMAR_TRIGGER_TYPE_WORD,854 "<|START_ACTION|>",855 });856 data.preserved_tokens = {857 "<|START_ACTION|>",858 "<|END_ACTION|>",859 "<|START_RESPONSE|>",860 "<|END_RESPONSE|>",861 "<|START_THINKING|>",862 "<|END_THINKING|>",863 };864 auto adjusted_messages = json::array();865 for (const auto & msg : inputs.messages) {866 auto has_reasoning_content = msg.contains("reasoning_content") && msg.at("reasoning_content").is_string();867 auto has_tool_calls = msg.contains("tool_calls") && msg.at("tool_calls").is_array();868 if (has_reasoning_content && has_tool_calls) {869 auto adjusted_message = msg;870 adjusted_message["tool_plan"] = msg.at("reasoning_content");871 adjusted_message.erase("reasoning_content");872 adjusted_messages.push_back(adjusted_message);873 } else {874 adjusted_messages.push_back(msg);875 }876 }877 data.prompt = apply(tmpl, adjusted_messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt, {});878 data.format = inputs.extract_reasoning ? COMMON_CHAT_FORMAT_COMMAND_R7B_EXTRACT_REASONING : COMMON_CHAT_FORMAT_COMMAND_R7B;879 return data;880}881static common_chat_msg common_chat_parse_command_r7b(const std::string & input, bool extract_reasoning) {882 static const std::regex thought_regex("(<\\|START_THINKING\\|>([\\s\\S]*?)<\\|END_THINKING\\|>)([\\s\\S]*)");883 static const std::regex action_regex("<\\|START_ACTION\\|>([\\s\\S]*?)<\\|END_ACTION\\|>");884 static const std::regex response_regex("(?:<\\|START_RESPONSE\\|>)?([\\s\\S]*?)<\\|END_RESPONSE\\|>");885 886 std::smatch match;887 888 common_chat_msg result;889 result.role = "assistant";890 891 std::string rest = input;892 893 if (std::regex_match(rest, match, thought_regex)) {894 if (extract_reasoning) {895 result.reasoning_content = match[2].str();896 } else if (!match[2].str().empty()) {897 // Let the unparsed thinking tags through in content only if their insides aren't empty.898 result.content = match[1].str();899 }900 rest = match[3].str();901 }902 if (std::regex_match(rest, match, action_regex)) {903 auto actions_str = match[1].str();904 auto actions = json::parse(actions_str);905 for (const auto & action : actions) {906 result.tool_calls.push_back({907 /* .name = */ action.at("tool_name"),908 /* .arguments = */ action.at("parameters").dump(),909 /* .id = */ action.at("tool_call_id"),910 });911 }912 } else if (std::regex_match(rest, match, response_regex)) {913 auto response = match[1].str();914 result.content += response;915 } else {916 result.content += rest;917 }918 return result;919}920 921static void expect_tool_parameters(const std::string & name, const json & parameters, const std::vector<std::string> & expected_properties) {922 if (!parameters.is_object() || !parameters.contains("type") || parameters.at("type") != "object" || !parameters.contains("properties") || !parameters.contains("required")) {923 throw std::runtime_error("Parameters of tool " + name + " must be an object w/ required properties");924 }925 const auto & parameters_properties = parameters.at("properties");926 const auto & parameters_required = parameters.at("required");927 for (const auto & prop : expected_properties) {928 if (!parameters_properties.contains(prop)) {929 throw std::runtime_error("Parameters of tool " + name + " is missing property: " + prop); // NOLINT930 }931 if (std::find(parameters_required.begin(), parameters_required.end(), json(prop)) == parameters_required.end()) {932 throw std::runtime_error("Parameters of tool " + name + " must have property marked as required: " + prop); // NOLINT933 }934 }935 if (parameters_properties.size() != expected_properties.size()) {936 throw std::runtime_error("Parameters of tool " + name + " must only have these properties:" + string_join(expected_properties, ", "));937 }938}939 940static common_chat_params common_chat_params_init_llama_3_1_tool_calls(const common_chat_template & tmpl, const struct templates_params & inputs, bool allow_python_tag_builtin_tools) {941 auto builtin_tools = json::array();942 common_chat_params data;943 data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;944 data.grammar = build_grammar([&](const common_grammar_builder & builder) {945 std::vector<std::string> tool_rules;946 947 auto handle_builtin_tool = [&](const std::string & name, const json & parameters) {948 if (name == "wolfram_alpha" || name == "web_search" || name == "brave_search") {949 // https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers/remote/tool_runtime/wolfram_alpha/wolfram_alpha.py950 // https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers/remote/tool_runtime/brave_search/brave_search.py951 expect_tool_parameters(name, parameters, {"query"});952 } else if (name == "python" || name == "code_interpreter") {953 // https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers/inline/tool_runtime/code_interpreter/code_interpreter.py954 expect_tool_parameters(name, parameters, {"code"});955 } else {956 return false;957 }958 959 std::vector<std::string> kvs;960 for (const auto & [key, value] : parameters.at("properties").items()) {961 kvs.push_back("\"" + key + "=\" " + builder.add_schema(name + "-args-" + key, value)); // NOLINT962 }963 964 tool_rules.push_back(965 builder.add_rule(966 name + "-call",967 "\"<|python_tag|>" + name + ".call(\" " + string_join(kvs, " \", \" ") + " \")\""));968 builtin_tools.push_back(name);969 970 return true;971 };972 973 foreach_function(inputs.tools, [&](const json & tool) {974 const auto & function = tool.at("function");975 std::string name = function.at("name");976 auto parameters = function.at("parameters");977 builder.resolve_refs(parameters);978 979 // https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/remote/tool_runtime980 if (allow_python_tag_builtin_tools) {981 handle_builtin_tool(name, parameters);982 }983 tool_rules.push_back(984 builder.add_rule(985 name + "-call",986 "\"{\" space "987 "( \"\\\"type\\\"\" space \":\" space \"\\\"function\\\"\" space \",\" space )? "988 " \"\\\"name\\\"\" space \":\" space \"\\\"" + name + "\\\"\" space \",\" space "989 " \"\\\"parameters\\\"\" space \":\" space " + builder.add_schema(name + "-args", parameters) + " "990 "\"}\" space"));991 });992 // Small models may hallucinate function names so we match anything (*at the start*) that looks like the JSON of a function call, regardless of the name.993 data.grammar_triggers.push_back({994 COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_START,995 "\\{\\s*(?:\"type\"\\s*:\\s*\"function\"\\s*,\\s*)?\"name\"\\s*:\\s*\"", // + name + "\"[\\s\\S]*",996 });997 if (!builtin_tools.empty()) {998 data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|python_tag|>"});999 data.preserved_tokens.push_back("<|python_tag|>");1000 }1001 // Allow a few empty lines on top of the usual constrained json schema space rule.1002 builder.add_rule("root", string_join(tool_rules, " | "));1003 });1004 data.additional_stops.push_back("<|eom_id|>");1005 data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt, {1006 {"tools_in_user_message", false},1007 {"builtin_tools", builtin_tools.empty() ? json() : builtin_tools},1008 });1009 data.format = allow_python_tag_builtin_tools && !builtin_tools.empty()1010 ? COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS1011 : COMMON_CHAT_FORMAT_LLAMA_3_X;1012 return data;1013}1014static common_chat_msg common_chat_parse_llama_3_1(const std::string & input, bool with_builtin_tools = false) {1015 // TODO: tighten & simplify the parser, don't accept leading text context.1016 static const std::regex function_regex(1017 "\\s*\\{\\s*(?:\"type\"\\s*:\\s*\"function\"\\s*,\\s*)?\"name\"\\s*:\\s*\"([^\"]+)\"\\s*,\\s*\"parameters\"\\s*: ");1018 static const std::regex close_regex("\\}\\s*");1019 static const std::regex builtin_call_regex("<\\|python_tag\\|>\\s*([^.(]+)\\s*\\.\\s*call\\s*\\(\\s*([\\w]+)\\s*=\\s*([\\s\\S]*?)\\)");1020 1021 if (with_builtin_tools) {1022 std::smatch match;1023 if (std::regex_match(input, match, builtin_call_regex)) {1024 try {1025 auto name = match[1].str();1026 auto arg_name = match[2].str();1027 auto arg_value_str = match[3].str();1028 auto arg_value = json::parse(arg_value_str);1029 1030 common_chat_msg msg;1031 msg.role = "assistant";1032 msg.tool_calls.push_back({1033 /* .name = */ name,1034 /* .arguments = */ (json {1035 {arg_name, arg_value},1036 }).dump(),1037 /* .id = */ "",1038 });1039 return msg;1040 } catch (const std::exception & e) {1041 LOG_WRN("Failed to parse builtin tool call arguments (%s): %s", e.what(), input.c_str());1042 }1043 }1044 }1045 return parse_json_tool_calls(input, std::nullopt, function_regex, close_regex);1046}1047 1048static common_chat_params common_chat_params_init_deepseek_r1(const common_chat_template & tmpl, const struct templates_params & inputs) {1049 common_chat_params data;1050 if (inputs.tools.is_array() && !inputs.tools.empty()) {1051 data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED && inputs.json_schema.is_null();1052 data.grammar = build_grammar([&](const common_grammar_builder & builder) {1053 std::vector<std::string> tool_rules;1054 foreach_function(inputs.tools, [&](const json & tool) {1055 const auto & function = tool.at("function");1056 std::string name = function.at("name");1057 auto parameters = function.at("parameters");1058 builder.resolve_refs(parameters);1059 tool_rules.push_back(builder.add_rule(name + "-call",1060 "\"<｜tool▁call▁begin｜>function<｜tool▁sep｜>" + name + "\\n"1061 "```json\\n\" " + builder.add_schema(name + "-args", parameters) + " "1062 "\"```<｜tool▁call▁end｜>\""));1063 });1064 // Distill Qwen 7B & 32B models seem confused re/ syntax of their tool call opening tag,1065 // so we accept common variants (then it's all constrained)1066 builder.add_rule("root",1067 "( \"<｜tool▁calls▁begin｜>\" | \"<｜tool_calls_begin｜>\" | \"<｜tool calls begin｜>\" | \"<｜tool\\\\_calls\\\\_begin｜>\" ) "1068 "(" + string_join(tool_rules, " | ") + ")" + (inputs.parallel_tool_calls ? "*" : "") + " "1069 "\"<｜tool▁calls▁end｜>\""1070 " space");1071 data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<｜tool▁calls▁begin｜>"});1072 data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<｜tool_calls_begin｜>"});1073 data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<｜tool calls begin｜>"});1074 data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<｜tool\\_calls\\_begin｜>"});1075 data.preserved_tokens = {1076 "<think>",1077 "</think>",1078 "<｜tool▁calls▁begin｜>",1079 "<｜tool▁call▁begin｜>",1080 "<｜tool▁sep｜>",1081 "<｜tool▁call▁end｜>",1082 "<｜tool▁calls▁end｜",1083 };1084 });1085 }1086 auto prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);1087 1088 // Hacks to fix the official (broken) prompt.1089 // It is advisable to use --chat-template-file models/templates/llama-cpp-deepseek-r1.jinja instead,1090 // until the official template is fixed.1091 if (tmpl.source().find("{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}") != std::string::npos) {1092 // Don't leave the chat dangling after tool results1093 if (string_ends_with(prompt, "<｜tool▁outputs▁end｜>")) {1094 prompt += "<｜end▁of▁sentence｜>";1095 if (inputs.add_generation_prompt) {1096 prompt += "<｜Assistant｜>";1097 }1098 }1099 // Fix up tool call delta example added by Minja1100 prompt = std::regex_replace(1101 prompt,1102 std::regex("(<｜tool▁call▁end｜>)[\\s\\r\\n]*(<｜tool▁outputs▁begin｜>|<｜User｜>)"),1103 "$1<｜tool▁calls▁end｜><｜end▁of▁sentence｜>$2");1104 }1105 data.prompt = prompt;1106 data.format = inputs.extract_reasoning ? COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING : COMMON_CHAT_FORMAT_DEEPSEEK_R1;1107 return data;1108}1109static common_chat_msg handle_think_tag_prelude(const std::string & input, bool extract_reasoning, const std::function<common_chat_msg(const std::string &)> & rest_parser) {1110 std::smatch match;1111 static const std::regex reasoning_content_regex("((?:<think>)?([\\s\\S\\r\\n]*?)</think>)?([\\s\\S\\r\\n]*)");1112 if (std::regex_match(input, match, reasoning_content_regex)) {1113 auto rest = match[3].str();1114 auto msg = rest_parser(rest);1115 auto reasoning_content = string_strip(match[2].str());1116 if (extract_reasoning) {1117 msg.reasoning_content = reasoning_content;1118 } else if (!reasoning_content.empty()) {1119 std::ostringstream content;1120 content << "<think>" << reasoning_content << "</think>" << msg.content;1121 msg.content = content.str();1122 }1123 return msg;1124 }1125 return rest_parser(input);1126}1127static common_chat_msg common_chat_parse_deepseek_r1(const std::string & input, bool extract_reasoning) {1128 return handle_think_tag_prelude(input, extract_reasoning, [](const std::string & input) {1129 static const std::regex function_regex("<｜tool▁call▁begin｜>function<｜tool▁sep｜>([^\n]+)\n```json\n");1130 static const std::regex close_regex("```[\\s\\r\\n]*<｜tool▁call▁end｜>");1131 static const std::regex tool_calls_regex("[\\s\\r\\n]*(?:<｜tool▁calls▁begin｜>|<｜tool_calls_begin｜>|<｜tool calls begin｜>|<｜tool\\\\_calls\\\\_begin｜>)([\\s\\S\\r\\n]*?)<｜tool▁calls▁end｜>");1132 1133 common_chat_msg msg;1134 msg.role = "assistant";1135 std::smatch match;1136 if (std::regex_search(input, match, tool_calls_regex)) {1137 auto tool_calls = match[1].str();1138 auto msg2 = parse_json_tool_calls(tool_calls, std::nullopt, function_regex, close_regex);1139 msg.tool_calls = std::move(msg2.tool_calls);1140 } else {1141 msg.content = input;1142 }1143 return msg;1144 });1145}1146 1147static common_chat_params common_chat_params_init_firefunction_v2(const common_chat_template & tmpl, const struct templates_params & inputs) {1148 LOG_DBG("%s\n", __func__);1149 common_chat_params data;1150 data.prompt = apply(tmpl, inputs.messages, /* tools= */ nullptr, inputs.add_generation_prompt, {1151 {"datetime", "Jan 29 2025 13:00:00 GMT"},1152 {"functions", json(inputs.tools.empty() ? "" : inputs.tools.dump(2))},1153 });1154 if (inputs.tools.is_array() && !inputs.tools.empty()) {1155 data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;1156 data.grammar = build_grammar([&](const common_grammar_builder & builder) {1157 auto schemas = json::array();1158 foreach_function(inputs.tools, [&](const json & tool) {1159 const auto & function = tool.at("function");1160 schemas.push_back({1161 {"type", "object"},1162 {"properties", {1163 {"name", {1164 {"type", "string"},1165 {"const", function.at("name")},1166 }},1167 {"arguments", function.at("parameters")},1168 }},1169 {"required", json::array({"name", "arguments", "id"})},1170 });1171 });1172 auto schema = json {1173 {"type", "array"},1174 {"items", schemas.size() == 1 ? schemas[0] : json {{"anyOf", schemas}}},1175 {"minItems", 1},1176 };1177 if (!inputs.parallel_tool_calls) {1178 schema["maxItems"] = 1;1179 }1180 builder.add_rule("root", "\" functools\"? " + builder.add_schema("tool_calls", schema));1181 });1182 data.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, " functools["});1183 data.preserved_tokens = {1184 " functools[",1185 };1186 data.format = COMMON_CHAT_FORMAT_FIREFUNCTION_V2;1187 } else {1188 data.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;1189 }1190 return data;1191}1192static common_chat_msg common_chat_parse_firefunction_v2(const std::string & input) {1193 return parse_prefixed_json_tool_call_array(input, " functools[", /* rstrip_prefix= */ 1);1194}1195 1196static common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, const struct templates_params & inputs) {1197 // >>>all\nlet's call functions>>>fn1\n{"arg1": 1...}\n>>>fn2\n{"arg1": 1...}...1198 // Using ">>>f1\n", ">>>f2\n"... as trigger words for the grammar1199 common_chat_params data;1200 data.prompt = apply(tmpl, inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);