Codeprocastinator/optimized-tinyllama-covalent
0119
1// Tests chat handling, including grammar generation and parsing for tool calling, for various templates.2//3// Also acts as a CLI to generate a Markdown summary of the formats of Jinja templates,4// e.g. given Minja (http://github.com/google/minja) checked out in parent dir:5//6// cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null7//8#include <fstream>9#include <iostream>10#include <json.hpp>11#include <string>12 13#include "chat.h"14#include "llama-grammar.h"15#include "unicode.h"16 17using json = nlohmann::ordered_json;18 19 20template <class T> static void assert_equals(const T & expected, const T & actual) {21 if (expected != actual) {22 std::cerr << "Expected: " << expected << std::endl;23 std::cerr << "Actual: " << actual << std::endl;24 std::cerr << std::flush;25 throw std::runtime_error("Test failed");26 }27}28 29static std::string read_file(const std::string & path) {30 std::cerr << "# Reading: " << path << '\n' << std::flush;31 std::ifstream fs(path, std::ios_base::binary);32 if (!fs.is_open()) {33 fs = std::ifstream("../" + path, std::ios_base::binary);34 if (!fs.is_open()) {35 throw std::runtime_error("Failed to open file: " + path);36 }37 }38 fs.seekg(0, std::ios_base::end);39 auto size = fs.tellg();40 fs.seekg(0);41 std::string out;42 out.resize(static_cast<size_t>(size));43 fs.read(out.data(), static_cast<std::streamsize>(size));44 return out;45}46 47static common_chat_templates_ptr read_templates(const std::string & path) {48 return common_chat_templates_ptr(common_chat_templates_init(/* model= */ nullptr, read_file(path)));49}50 51static std::unique_ptr<llama_grammar> build_grammar(const std::string & grammar_str) {52 return std::unique_ptr<llama_grammar>(53 llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0));54}55 56// TODO: extract to common helper (copied from test-grammar-integration.cpp)57static bool match_string(const std::string & input, llama_grammar * grammar) {58 const auto cpts = unicode_cpts_from_utf8(input);59 60 auto & stacks_cur = llama_grammar_get_stacks(grammar);61 62 for (const auto & cpt : cpts) {63 llama_grammar_accept(grammar, cpt);64 65 if (stacks_cur.empty()) {66 // no stacks means that the grammar failed to match at this point67 return false;68 }69 }70 71 if (std::any_of(stacks_cur.begin(), stacks_cur.end(), [](const auto & stack) { return stack.empty(); })) {72 // An empty stack means that the grammar has been completed73 return true;74 }75 76 return false;77}78 79static void assert_msg_equals(const common_chat_msg & expected, const common_chat_msg & actual) {80 assert_equals(expected.role, actual.role);81 assert_equals(expected.content, actual.content);82 assert_equals(expected.content_parts.size(), actual.content_parts.size());83 for (size_t i = 0; i < expected.content_parts.size(); i++) {84 const auto & expected_part = expected.content_parts[i];85 const auto & actual_part = actual.content_parts[i];86 assert_equals(expected_part.type, actual_part.type);87 assert_equals(expected_part.text, actual_part.text);88 }89 assert_equals(expected.reasoning_content, actual.reasoning_content);90 assert_equals(expected.tool_calls.size(), actual.tool_calls.size());91 for (size_t i = 0; i < expected.tool_calls.size(); i++) {92 const auto & expected_tool_call = expected.tool_calls[i];93 const auto & actual_tool_call = actual.tool_calls[i];94 assert_equals(expected_tool_call.name, actual_tool_call.name);95 assert_equals(json::parse(expected_tool_call.arguments).dump(), json::parse(actual_tool_call.arguments).dump());96 assert_equals(expected_tool_call.id, actual_tool_call.id);97 }98}99 100common_chat_tool special_function_tool {101 /* .name = */ "special_function",102 /* .description = */ "I'm special",103 /* .parameters = */ R"({104 "type": "object",105 "properties": {106 "arg1": {107 "type": "integer",108 "description": "The arg."109 }110 },111 "required": ["arg1"]112 })",113};114common_chat_tool python_tool {115 /* .name = */ "python",116 /* .description = */ "an ipython interpreter",117 /* .parameters = */ R"({118 "type": "object",119 "properties": {120 "code": {121 "type": "string",122 "description": "Python code to execute."123 }124 },125 "required": ["code"]126 })",127};128common_chat_tool code_interpreter_tool {129 /* .name = */ "code_interpreter",130 /* .description = */ "an ipython interpreter",131 /* .parameters = */ R"({132 "type": "object",133 "properties": {134 "code": {135 "type": "string",136 "description": "Python code to execute."137 }138 },139 "required": ["code"]140 })",141};142std::vector<common_chat_tool> tools { special_function_tool, python_tool };143std::vector<common_chat_tool> llama_3_1_tools { special_function_tool, code_interpreter_tool };144 145struct delta_data {146 std::string delta;147 common_chat_params params;148};149 150static delta_data init_delta(const struct common_chat_templates * tmpls, const std::vector<std::string> & end_tokens,151 const common_chat_msg & user_message,152 const common_chat_msg & delta_message,153 const std::vector<common_chat_tool> & tools,154 const common_chat_tool_choice & tool_choice,155 bool think = false) {156 common_chat_templates_inputs inputs;157 inputs.parallel_tool_calls = true;158 inputs.messages.push_back(user_message);159 inputs.tools = tools;160 inputs.tool_choice = tool_choice;161 inputs.extract_reasoning = think;162 auto params_prefix = common_chat_templates_apply(tmpls, inputs);163 164 inputs.messages.push_back(delta_message);165 inputs.add_generation_prompt = false;166 auto params_full = common_chat_templates_apply(tmpls, inputs);167 168 std::string prefix = params_prefix.prompt;169 std::string full = params_full.prompt;170 171 if (full == prefix) {172 throw std::runtime_error("Full message is the same as the prefix");173 }174 175 size_t common_prefix_length = 0;176 for (size_t i = 0; i < prefix.size() && i < full.size(); ++i) {177 if (prefix[i] != full[i]) {178 break;179 }180 if (prefix[i] == '<') {181 // DeepSeek R1's template (as of 20250209) adds a trailing <think> if add_generation_prompt,182 // but it removes thinking tags for past messages.183 // The prefix and full strings diverge at <think> vs. <｜tool▁calls▁begin｜>, we avoid consuming the leading <.184 continue;185 }186 common_prefix_length = i + 1;187 }188 auto delta = full.substr(common_prefix_length);189 190 // Strip end tokens191 for (const auto & end_token : end_tokens) {192 // rfind to find the last occurrence193 auto pos = delta.rfind(end_token);194 if (pos != std::string::npos) {195 delta = delta.substr(0, pos);196 break;197 }198 }199 return { delta, params_full };200}201 202/*203 Applies the template to 1 user message w/ add_generation_prompt=true, then w/ the test message w/ add_generation_prompt=false,204 gets the diff, removes any end tokens and parses the result w/ the grammar, checking that205 the parsed message is the same as the test_message206*/207static void test_templates(const struct common_chat_templates * tmpls, const std::vector<std::string> & end_tokens,208 const common_chat_msg & test_message,209 const std::vector<common_chat_tool> & tools = {},210 const std::string & expected_delta = "",211 bool expect_grammar_triggered = true,212 bool test_grammar_if_triggered = true,213 bool think = false) {214 common_chat_msg user_message;215 user_message.role = "user";216 user_message.content = "Hello, world!";217 218 for (const auto & tool_choice : std::vector<common_chat_tool_choice> {COMMON_CHAT_TOOL_CHOICE_AUTO, COMMON_CHAT_TOOL_CHOICE_REQUIRED}) {219 auto data = init_delta(tmpls, end_tokens, user_message, test_message, tools, tool_choice, think);220 if (!expected_delta.empty()) {221 assert_equals(expected_delta, data.delta);222 }223 224 if (expect_grammar_triggered) {225 const auto msg = common_chat_parse(data.delta, data.params.format);226 assert_msg_equals(test_message, msg);227 }228 229 if (!test_message.tool_calls.empty()) {230 GGML_ASSERT(!data.params.grammar.empty());231 }232 if (!data.params.grammar.empty()) {233 auto grammar = build_grammar(data.params.grammar);234 if (!grammar) {235 throw std::runtime_error("Failed to build grammar");236 }237 auto earliest_trigger_pos = std::string::npos;238 auto constrained = data.delta;239 for (const auto & trigger : data.params.grammar_triggers) {240 size_t pos = std::string::npos;241 std::smatch match;242 switch (trigger.type) {243 case COMMON_GRAMMAR_TRIGGER_TYPE_WORD:244 {245 const auto & word = trigger.value;246 pos = constrained.find(word);247 break;248 }249 case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN:250 {251 const auto & pattern = trigger.value;252 if (std::regex_search(constrained, match, std::regex(pattern))) {253 pos = match.position();254 }255 break;256 }257 case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_START:258 {259 const auto & pattern = trigger.value;260 if (std::regex_search(constrained, match, std::regex(pattern)) && match.position() == 0) {261 pos = 0;262 }263 break;264 }265 default:266 throw std::runtime_error("Unknown trigger type");267 }268 if (pos == std::string::npos) {269 continue;270 }271 if (earliest_trigger_pos == std::string::npos || pos < earliest_trigger_pos) {272 earliest_trigger_pos = pos;273 }274 }275 auto grammar_triggered = false;276 if (earliest_trigger_pos != std::string::npos) {277 constrained = constrained.substr(earliest_trigger_pos);278 grammar_triggered = true;279 }280 if (data.params.grammar_lazy) {281 assert_equals(expect_grammar_triggered, grammar_triggered);282 }283 284 if (grammar_triggered && test_grammar_if_triggered && !match_string(constrained, grammar.get())) {285 throw std::runtime_error("Failed to match delta against grammar:\n\n" + data.delta +286 "\n\nConstrained: " + constrained +287 "\n\nGrammar: " + data.params.grammar);288 }289 }290 }291}292 293const common_chat_msg message_user {294 "user",295 "Hey there!",296 /* .content_parts = */ {},297 /* .tool_calls = */ {},298 /* .reasoning_content = */ "",299 /* .tool_name = */ "",300 /* .tool_call_id = */ "",301};302 303const common_chat_msg message_user_parts {304 "user",305 /* .content = */ "",306 /* .content_parts = */ {307 { "text", "Hey" },308 { "text", "there" },309 },310 /* .tool_calls = */ {},311 /* .reasoning_content = */ "",312 /* .tool_name = */ "",313 /* .tool_call_id = */ "",314};315const common_chat_msg message_assist {316 "assistant",317 "Hello, world!\nWhat's up?",318 /* .content_parts = */ {},319 /* .tool_calls = */ {},320 /* .reasoning_content = */ "",321 /* .tool_name = */ "",322 /* .tool_call_id = */ "",323};324const common_chat_msg message_assist_thoughts_unparsed_think {325 "assistant",326 "<think>I'm thinking</think>Hello, world!\nWhat's up?",327 /* .content_parts = */ {},328 /* .tool_calls = */ {},329 /* .reasoning_content = */ "",330 /* .tool_name = */ "",331 /* .tool_call_id = */ "",332};333const common_chat_msg message_assist_thoughts_unparsed_r7b {334 "assistant",335 "<|START_THINKING|>I'm thinking<|END_THINKING|>Hello, world!\nWhat's up?",336 /* .content_parts = */ {},337 /* .tool_calls = */ {},338 /* .reasoning_content = */ "",339 /* .tool_name = */ "",340 /* .tool_call_id = */ "",341};342const common_chat_msg message_assist_thoughts {343 "assistant",344 "Hello, world!\nWhat's up?",345 /* .content_parts = */ {},346 /* .tool_calls = */ {},347 /* .reasoning_content = */ "I'm thinking",348 /* .tool_name = */ "",349 /* .tool_call_id = */ "",350};351const std::vector<common_chat_tool_call> tool_calls {352 { "special_function", "{\"arg1\": 1}", /* .id = */ "" },353};354const std::vector<common_chat_tool_call> tool_calls_idx {355 { "special_function", "{\"arg1\": 1}", /* .id = */ "0" },356};357const std::vector<common_chat_tool_call> tool_calls_id {358 { "special_function", "{\"arg1\": 1}", /* .id = */ "123456789" },359};360 361const common_chat_msg message_assist_call {362 "assistant",363 "",364 /* .content_parts = */ {},365 tool_calls,366 /* .reasoning_content = */ "",367 /* .tool_name = */ "",368 /* .tool_call_id = */ "",369};370const common_chat_msg message_assist_call_thoughts = {371 "assistant",372 /* .content = */ "",373 /* .content_parts = */ {},374 tool_calls,375 /* .reasoning_content = */ "I'm\nthinking",376 /* .tool_name = */ "",377 /* .tool_call_id = */ "",378};379const common_chat_msg message_assist_call_thoughts_unparsed = {380 "assistant",381 /* .content = */ "<think>I'm\nthinking</think>",382 /* .content_parts = */ {},383 tool_calls,384 /* .reasoning_content = */ "",385 /* .tool_name = */ "",386 /* .tool_call_id = */ "",387};388const common_chat_msg message_assist_call_id {389 "assistant",390 "",391 /* .content_parts = */ {},392 tool_calls_id,393 /* .reasoning_content = */ "",394 /* .tool_name = */ "",395 /* .tool_call_id = */ "",396};397const common_chat_msg message_assist_call_idx {398 "assistant",399 "",400 /* .content_parts = */ {},401 tool_calls_idx,402 /* .reasoning_content = */ "",403 /* .tool_name = */ "",404 /* .tool_call_id = */ "",405};406const common_chat_msg message_assist_call_python {407 "assistant",408 "",409 /* .content_parts = */ {},410 { { "python", "{\"code\": \"print('hey')\"}", /* .id = */ "" } },411 /* .reasoning_content = */ "",412 /* .tool_name = */ "",413 /* .tool_call_id = */ "",414};415const common_chat_msg message_assist_call_code_interpreter {416 "assistant",417 "",418 /* .content_parts = */ {},419 { { "code_interpreter", "{\"code\": \"print('hey')\"}", /* .id = */ "" } },420 /* .reasoning_content = */ "",421 /* .tool_name = */ "",422 /* .tool_call_id = */ "",423};424 425static void test_msgs_oaicompat_json_conversion() {426 std::vector<common_chat_msg> msgs{427 message_user,428 message_user_parts,429 message_assist_call,430 message_assist_call_thoughts,431 message_assist_call_thoughts_unparsed,432 message_assist_call_id,433 message_assist_call_idx,434 message_assist_call_python,435 message_assist_call_code_interpreter,436 };437 for (const auto & msg : msgs) {438 auto oai_json = common_chat_msgs_to_json_oaicompat<json>({msg});439 auto msgs2 = common_chat_msgs_parse_oaicompat(oai_json);440 assert_equals((size_t) 1, msgs2.size());441 auto msg2 = msgs2[0];442 assert_msg_equals(msg, msg2);443 }444 assert_equals(445 std::string(446 "[\n"447 " {\n"448 " \"role\": \"user\",\n"449 " \"content\": [\n"450 " {\n"451 " \"type\": \"text\",\n"452 " \"text\": \"Hey\"\n"453 " },\n"454 " {\n"455 " \"type\": \"text\",\n"456 " \"text\": \"there\"\n"457 " }\n"458 " ]\n"459 " }\n"460 "]"461 ),462 common_chat_msgs_to_json_oaicompat<json>({message_user_parts}).dump(2));463 464 assert_equals(465 std::string(466 "[\n"467 " {\n"468 " \"role\": \"assistant\",\n"469 " \"content\": null,\n"470 " \"tool_calls\": [\n"471 " {\n"472 " \"type\": \"function\",\n"473 " \"function\": {\n"474 " \"name\": \"python\",\n"475 " \"arguments\": \"{\\\"code\\\": \\\"print('hey')\\\"}\"\n"476 " }\n"477 " }\n"478 " ]\n"479 " }\n"480 "]"481 ),482 common_chat_msgs_to_json_oaicompat<json>({message_assist_call_python}).dump(2));483 484 auto res = common_chat_msgs_parse_oaicompat(json::parse("[{\"role\": \"assistant\", \"tool_calls\": []}]"));485 assert_equals<size_t>(1, res.size());486 assert_equals<std::string>(res[0].role, "assistant");487 assert_equals(true, res[0].content.empty());488 assert_equals(true, res[0].tool_calls.empty());489 490 try {491 common_chat_msgs_parse_oaicompat(json::parse("[{\"role\": \"assistant\"}]"));492 throw std::runtime_error("Expected exception");493 } catch (const std::exception & e) {494 if (std::string(e.what()).find("'content'") == std::string::npos) {495 throw std::runtime_error("Expected exception about missing 'content'");496 }497 }498}499 500static void test_tools_oaicompat_json_conversion() {501 std::vector<common_chat_tool> tools{502 special_function_tool,503 python_tool,504 code_interpreter_tool,505 };506 507 for (const auto & tool : tools) {508 auto oai_json = common_chat_tools_to_json_oaicompat<json>({tool});509 auto tools2 = common_chat_tools_parse_oaicompat(oai_json);510 assert_equals((size_t) 1, tools2.size());511 auto tool2 = tools2[0];512 assert_equals(tool.name, tool2.name);513 assert_equals(tool.description, tool2.description);514 assert_equals(json::parse(tool.parameters).dump(2), json::parse(tool2.parameters).dump(2));515 }516 517 assert_equals(518 std::string(519 "[\n"520 " {\n"521 " \"type\": \"function\",\n"522 " \"function\": {\n"523 " \"name\": \"special_function\",\n"524 " \"description\": \"I'm special\",\n"525 " \"parameters\": {\n"526 " \"type\": \"object\",\n"527 " \"properties\": {\n"528 " \"arg1\": {\n"529 " \"type\": \"integer\",\n"530 " \"description\": \"The arg.\"\n"531 " }\n"532 " },\n"533 " \"required\": [\n"534 " \"arg1\"\n"535 " ]\n"536 " }\n"537 " }\n"538 " }\n"539 "]"540 ),541 common_chat_tools_to_json_oaicompat<json>({special_function_tool}).dump(2));542}543 544static void test_template_output_parsers() {545 546 common_chat_templates_inputs inputs_no_tools;547 inputs_no_tools.messages = {message_user};548 inputs_no_tools.extract_reasoning = false;549 550 common_chat_templates_inputs inputs_no_tools_think;551 inputs_no_tools_think.messages = {message_user};552 inputs_no_tools_think.extract_reasoning = true;553 554 common_chat_templates_inputs inputs_tools;555 inputs_tools.messages = {message_user};556 inputs_tools.tools = {special_function_tool};557 inputs_tools.extract_reasoning = false;558 559 common_chat_templates_inputs inputs_tools_think;560 inputs_tools_think.messages = {message_user};561 inputs_tools_think.tools = {special_function_tool};562 inputs_tools_think.extract_reasoning = true;563 564 common_chat_templates_inputs inputs_tools_builtin;565 inputs_tools_builtin.messages = {message_user};566 inputs_tools_builtin.tools = {python_tool};567 inputs_tools_builtin.extract_reasoning = false;568 569 {570 // Not supported yet571 auto tmpls = read_templates("models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja");572 assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);573 assert_equals(COMMON_CHAT_FORMAT_GENERIC, common_chat_templates_apply(tmpls.get(), inputs_tools).format);574 }575 {576 auto tmpls = read_templates("models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja");577 std::vector<std::string> end_tokens{ "<|END_OF_TURN_TOKEN|>" };578 579 assert_equals(COMMON_CHAT_FORMAT_COMMAND_R7B, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);580 assert_equals(COMMON_CHAT_FORMAT_COMMAND_R7B, common_chat_templates_apply(tmpls.get(), inputs_tools).format);581 assert_equals(COMMON_CHAT_FORMAT_COMMAND_R7B_EXTRACT_REASONING, common_chat_templates_apply(tmpls.get(), inputs_tools_think).format);582 583 assert_msg_equals(message_assist,584 common_chat_parse(585 "Hello, world!\nWhat's up?",586 COMMON_CHAT_FORMAT_COMMAND_R7B));587 assert_msg_equals(message_assist,588 common_chat_parse(589 "Hello, world!\nWhat's up?<|END_RESPONSE|>",590 COMMON_CHAT_FORMAT_COMMAND_R7B));591 assert_msg_equals(message_assist,592 common_chat_parse(593 "<|START_RESPONSE|>Hello, world!\nWhat's up?<|END_RESPONSE|>",594 COMMON_CHAT_FORMAT_COMMAND_R7B));595 assert_msg_equals(message_assist_thoughts_unparsed_r7b,596 common_chat_parse(597 "<|START_THINKING|>I'm thinking<|END_THINKING|>"598 "<|START_RESPONSE|>Hello, world!\nWhat's up?<|END_RESPONSE|>",599 COMMON_CHAT_FORMAT_COMMAND_R7B));600 assert_msg_equals(message_assist_thoughts_unparsed_r7b,601 common_chat_parse(602 "<|START_THINKING|>I'm thinking<|END_THINKING|>"603 "Hello, world!\nWhat's up?<|END_RESPONSE|>",604 COMMON_CHAT_FORMAT_COMMAND_R7B));605 606 assert_msg_equals(message_assist_thoughts,607 common_chat_parse(608 "<|START_THINKING|>I'm thinking<|END_THINKING|>"609 "<|START_RESPONSE|>Hello, world!\nWhat's up?<|END_RESPONSE|>",610 COMMON_CHAT_FORMAT_COMMAND_R7B_EXTRACT_REASONING));611 612 test_templates(tmpls.get(), end_tokens, message_assist_call_idx, tools,613 "<|START_THINKING|><|END_THINKING|>"614 "<|START_ACTION|>[\n"615 " {\"tool_call_id\": \"0\", \"tool_name\": \"special_function\", \"parameters\": {\"arg1\": 1}}\n"616 "]<|END_ACTION|>");617 test_templates(tmpls.get(), end_tokens, message_assist, tools,618 "<|START_RESPONSE|>Hello, world!\n"619 "What's up?<|END_RESPONSE|>",620 /* expect_grammar_triggered= */ false);621 }622 {623 auto tmpls = read_templates("models/templates/google-gemma-2-2b-it.jinja");624 std::vector<std::string> end_tokens{ "<end_of_turn>" };625 626 assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);627 assert_equals(COMMON_CHAT_FORMAT_GENERIC, common_chat_templates_apply(tmpls.get(), inputs_tools).format);628 assert_equals(COMMON_CHAT_FORMAT_GENERIC,629 common_chat_templates_apply(630 read_templates("models/templates/microsoft-Phi-3.5-mini-instruct.jinja").get(),631 inputs_tools)632 .format);633 634 // Generic tool calls doesn't generate / parse content-only messages symmetrically.635 636 assert_msg_equals(message_assist,637 common_chat_parse("{\n"638 " \"response\": \"Hello, world!\\nWhat's up?\"\n"639 "}",640 common_chat_templates_apply(tmpls.get(), inputs_tools).format));641 test_templates(tmpls.get(), end_tokens, message_assist_call_id, tools,642 "{\n"643 " \"tool_calls\": [\n"644 " {\n"645 " \"name\": \"special_function\",\n"646 " \"arguments\": {\n"647 " \"arg1\": 1\n"648 " },\n"649 " \"id\": \"123456789\"\n"650 " }\n"651 " ]\n"652 "}");653 }654 {655 auto tmpls = read_templates("models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja");656 std::vector<std::string> end_tokens{ "</s>" };657 658 assert_equals(COMMON_CHAT_FORMAT_MISTRAL_NEMO, common_chat_templates_apply(tmpls.get(), inputs_tools).format);659 660 test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);661 test_templates(662 tmpls.get(), end_tokens, message_assist_call_id, tools,663 "[TOOL_CALLS][{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}, \"id\": \"123456789\"}]");664 }665 {666 auto tmpls = read_templates("models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja");667 std::vector<std::string> end_tokens{ "<|im_end|>" };668 669 assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);670 assert_equals(COMMON_CHAT_FORMAT_HERMES_2_PRO, common_chat_templates_apply(tmpls.get(), inputs_tools).format);671 assert_equals(672 COMMON_CHAT_FORMAT_HERMES_2_PRO,673 common_chat_templates_apply(674 read_templates("models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja").get(),675 inputs_tools)676 .format);677 assert_equals(678 COMMON_CHAT_FORMAT_HERMES_2_PRO,679 common_chat_templates_apply(680 read_templates("models/templates/Qwen-Qwen2.5-7B-Instruct.jinja").get(),681 inputs_tools)682 .format);683 684 // Test parsing685 assert_msg_equals(message_assist_call, common_chat_parse(686 "<tool_call>\n"687 "{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"688 "</tool_call>",689 COMMON_CHAT_FORMAT_HERMES_2_PRO));690 assert_msg_equals(message_assist_call, common_chat_parse(691 "<function=special_function>{\"arg1\": 1}</function>",692 COMMON_CHAT_FORMAT_HERMES_2_PRO));693 assert_msg_equals(message_assist_call, common_chat_parse(694 "<function name=\"special_function\">\n"695 "{\"arg1\": 1}\n"696 "</function>",697 COMMON_CHAT_FORMAT_HERMES_2_PRO));698 assert_msg_equals(message_assist_call, common_chat_parse(699 "<tool>\n"700 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"701 "</tool>",702 COMMON_CHAT_FORMAT_HERMES_2_PRO));703 assert_msg_equals(message_assist_call, common_chat_parse(704 "<tools>\n"705 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"706 "</tools>",707 COMMON_CHAT_FORMAT_HERMES_2_PRO));708 assert_msg_equals(message_assist_call, common_chat_parse(709 "<response>\n"710 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"711 "</response>",712 COMMON_CHAT_FORMAT_HERMES_2_PRO));713 assert_msg_equals(message_assist_call, common_chat_parse(714 "```xml\n"715 "<response>\n"716 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"717 "</response>\n"718 "```",719 COMMON_CHAT_FORMAT_HERMES_2_PRO));720 assert_msg_equals(message_assist_call, common_chat_parse(721 "```xml\n"722 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"723 "```",724 COMMON_CHAT_FORMAT_HERMES_2_PRO));725 assert_msg_equals(message_assist_call, common_chat_parse(726 "```\n"727 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"728 "```",729 COMMON_CHAT_FORMAT_HERMES_2_PRO));730 assert_msg_equals(message_assist_call, common_chat_parse(731 "```\n"732 "{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"733 "```",734 COMMON_CHAT_FORMAT_HERMES_2_PRO));735 assert_msg_equals(message_assist_call, common_chat_parse(736 "```json\n"737 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"738 "```",739 COMMON_CHAT_FORMAT_HERMES_2_PRO));740 assert_msg_equals(message_assist_call, common_chat_parse(741 "```json\n"742 "\n"743 " <function_call> {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}} \n"744 " </function_call> \n"745 "``` ",746 COMMON_CHAT_FORMAT_HERMES_2_PRO));747 assert_msg_equals(message_assist_call, common_chat_parse(748 "<json>\n"749 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"750 "</json>",751 COMMON_CHAT_FORMAT_HERMES_2_PRO));752 assert_msg_equals(message_assist_call, common_chat_parse(753 "<xml>\n"754 " {\n"755 " \"name\": \"special_function\", \"arguments\": {\"arg1\": 1}\n"756 " }\n"757 "</xml>",758 COMMON_CHAT_FORMAT_HERMES_2_PRO));759 assert_msg_equals(message_assist_call, common_chat_parse(760 "<JSON>\n"761 " {\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"762 "</JSON>",763 COMMON_CHAT_FORMAT_HERMES_2_PRO));764 assert_msg_equals(message_assist_call, common_chat_parse(765 "{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}",766 COMMON_CHAT_FORMAT_HERMES_2_PRO));767 assert_msg_equals(message_assist_call, common_chat_parse(768 "{\n \"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}",769 COMMON_CHAT_FORMAT_HERMES_2_PRO));770 771 assert_msg_equals(message_assist_thoughts_unparsed_think,772 common_chat_parse("<think>I'm thinking</think>Hello, world!\nWhat's up?",773 COMMON_CHAT_FORMAT_HERMES_2_PRO));774 assert_msg_equals(message_assist_thoughts_unparsed_think,775 common_chat_parse("I'm thinking</think>Hello, world!\nWhat's up?",776 COMMON_CHAT_FORMAT_HERMES_2_PRO));777 assert_msg_equals(message_assist_thoughts,778 common_chat_parse("<think>I'm thinking</think>Hello, world!\nWhat's up?",779 COMMON_CHAT_FORMAT_HERMES_2_PRO_EXTRACT_REASONING));780 assert_msg_equals(message_assist_thoughts,781 common_chat_parse("I'm thinking</think>Hello, world!\nWhat's up?",782 COMMON_CHAT_FORMAT_HERMES_2_PRO_EXTRACT_REASONING));783 784 test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);785 test_templates(tmpls.get(), end_tokens, message_assist_call, tools,786 "<tool_call>\n"787 "{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"788 "</tool_call>");789 test_templates(tmpls.get(), end_tokens, message_assist_call_python, tools,790 "<tool_call>\n"791 "{\"name\": \"python\", \"arguments\": {\"code\": \"print('hey')\"}}\n"792 "</tool_call>");793 }794 {795 auto tmpls = read_templates("models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja");796 std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };797 798 assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);799 assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X, common_chat_templates_apply(tmpls.get(), inputs_tools).format);800 assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS,801 common_chat_templates_apply(tmpls.get(), inputs_tools_builtin).format);802 assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS,803 common_chat_templates_apply(804 read_templates("models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja").get(),805 inputs_tools_builtin)806 .format);807 808 // test_templates(tmpls.get(), end_tokens, message_assist, tools, R"(?)", /* expect_grammar_triggered= */ false);809 test_templates(tmpls.get(), end_tokens, message_assist_call_code_interpreter, llama_3_1_tools,810 "<|python_tag|>code_interpreter.call(code=\"print('hey')\")");811 test_templates(tmpls.get(), end_tokens, message_assist_call_python, tools,812 "<|python_tag|>python.call(code=\"print('hey')\")");813 test_templates(tmpls.get(), end_tokens, message_assist_call, tools,814 "{\"name\": \"special_function\", \"parameters\": {\"arg1\": 1}}");815 }816 {817 auto tmpls = read_templates("models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja");818 std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };819 820 assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X, common_chat_templates_apply(tmpls.get(), inputs_tools).format);821 assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);822 823 test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);824 test_templates(tmpls.get(), end_tokens, message_assist_call, tools,825 "{\"name\": \"special_function\", \"parameters\": {\"arg1\": 1}}");826 }827 {828 auto tmpls = read_templates("models/templates/meetkai-functionary-medium-v3.1.jinja");829 std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };830 831 assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY,832 common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);833 assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1,834 common_chat_templates_apply(tmpls.get(), inputs_tools).format);835 836 test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);837 test_templates(tmpls.get(), end_tokens, message_assist_call, tools,838 "<function=special_function>{\"arg1\": 1}</function>");839 }840 {841 auto tmpls = read_templates("models/templates/meetkai-functionary-medium-v3.2.jinja");842 std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };843 844 assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);845 assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2, common_chat_templates_apply(tmpls.get(), inputs_tools).format);846 847 test_templates(tmpls.get(), end_tokens, message_assist, {},848 "all\n"849 "Hello, world!\n"850 "What's up?",851 /* expect_grammar_triggered= */ false);852 test_templates(tmpls.get(), end_tokens, message_assist_call, tools,853 "special_function\n"854 "{\"arg1\": 1}");855 }856 {857 auto tmpls = read_templates("models/templates/fireworks-ai-llama-3-firefunction-v2.jinja");858 std::vector<std::string> end_tokens{ "<|eot_id|>" };859 860 assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);861 assert_equals(COMMON_CHAT_FORMAT_FIREFUNCTION_V2, common_chat_templates_apply(tmpls.get(), inputs_tools).format);862 863 test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);864 test_templates(tmpls.get(), end_tokens, message_assist_call, tools,865 " functools[{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}]");866 }867 {868 // Original DeepSeek R1 template. Leaves <｜tool▁calls▁begin｜> and others unclosed. Our logic fixes the prompt.869 auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja");870 std::vector<std::string> end_tokens{ "<｜end▁of▁sentence｜>" };871 872 assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);873 assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1, common_chat_templates_apply(tmpls.get(), inputs_tools).format);874 assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING, common_chat_templates_apply(tmpls.get(), inputs_tools_think).format);875 876 test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);877 test_templates(tmpls.get(), end_tokens, message_assist_thoughts, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);878 assert_msg_equals(message_assist_thoughts_unparsed_think,879 common_chat_parse("<think>I'm thinking</think>Hello, world!\nWhat's up?",880 COMMON_CHAT_FORMAT_DEEPSEEK_R1));881 assert_msg_equals(message_assist_thoughts,882 common_chat_parse("<think>I'm thinking</think>Hello, world!\nWhat's up?",883 COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING));884 assert_msg_equals(message_assist_thoughts,885 // Latest template update (ast of 20250209) adds a trailing <think>\n if add_generation_prompt is true.886 common_chat_parse("I'm thinking</think>Hello, world!\nWhat's up?",887 COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING));888 // test_templates(tmpls.get(), end_tokens, message_assist_call, tools,889 // "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n"890 // "```json\n"891 // "{\"arg1\": 1}\n"892 // // Look what's not here: <｜tool▁calls▁end｜> (also missing the <｜end▁of▁sentence｜>, but that is removed lazily by the test's delta logic)893 // "```<｜tool▁call▁end｜>",894 // /* expect_grammar_triggered= */ true,895 // /* test_grammar_if_triggered= */ false);896 }897 {898 // Replacement DeepSeek R1 template. Makes the Distill Qwen 7B/32B models happy to call tools and all.899 auto tmpls = read_templates("models/templates/llama-cpp-deepseek-r1.jinja");900 std::vector<std::string> end_tokens{ "<｜end▁of▁sentence｜>" };901 902 assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1, common_chat_templates_apply(tmpls.get(), inputs_no_tools).format);903 assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1, common_chat_templates_apply(tmpls.get(), inputs_tools).format);904 assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING, common_chat_templates_apply(tmpls.get(), inputs_tools_think).format);905 906 test_templates(tmpls.get(), end_tokens, message_assist, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);907 test_templates(tmpls.get(), end_tokens, message_assist_thoughts, tools, "Hello, world!\nWhat's up?", /* expect_grammar_triggered= */ false);908 assert_msg_equals(message_assist_thoughts_unparsed_think,909 common_chat_parse("<think>I'm thinking</think>Hello, world!\nWhat's up?",910 COMMON_CHAT_FORMAT_DEEPSEEK_R1));911 assert_msg_equals(message_assist_thoughts,912 common_chat_parse("<think>I'm thinking</think>Hello, world!\nWhat's up?",913 COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING));914 915 assert_msg_equals(message_assist_call_thoughts_unparsed,916 common_chat_parse(917 "<think>I'm\nthinking</think>\n\n"918 "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n"919 "```json\n"920 "{\"arg1\": 1}\n"921 "```<｜tool▁call▁end｜><｜tool▁calls▁end｜>",922 COMMON_CHAT_FORMAT_DEEPSEEK_R1));923 assert_msg_equals(message_assist_call_thoughts,924 common_chat_parse(925 "<think>I'm\nthinking</think>\n\n"926 "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n"927 "```json\n"928 "{\"arg1\": 1}\n"929 "```<｜tool▁call▁end｜><｜tool▁calls▁end｜>",930 COMMON_CHAT_FORMAT_DEEPSEEK_R1_EXTRACT_REASONING));931 test_templates(tmpls.get(), end_tokens, message_assist_call, tools,932 "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n"933 "```json\n"934 "{\"arg1\": 1}\n"935 "```<｜tool▁call▁end｜><｜tool▁calls▁end｜>");936 }937}938 939int main(int argc, char ** argv) {940 // try {941#ifndef _WIN32942 if (argc > 1) {943 common_chat_templates_inputs inputs;944 common_chat_msg msg;945 msg.role = "user";946 msg.content = "Hey";947 inputs.messages = {msg};948 inputs.tools = { special_function_tool };949 950 std::cout << "| Template | Format |\n";951 std::cout << "|----------|--------|\n";952 953 for (int i = 1; i < argc; i++) {954 try {955 std::string path = argv[i];956 if (path.rfind(".jinja") != path.size() - 6) {957 std::cerr << "Skipping non-jinja file: " << path << '\n';958 continue;959 }960 auto tmpls = read_templates(path);961 auto parts = string_split(path, "/");962 auto name = parts[parts.size() - 1];963 auto format = common_chat_format_name(common_chat_templates_apply(tmpls.get(), inputs).format);964 std::cout << "| " << name << " | " << format << " |\n";965 } catch (const std::exception & e) {966 std::cerr << "Failed to process " << argv[i] << ": " << e.what() << '\n';967 }968 }969 } else970#endif971 {972 test_msgs_oaicompat_json_conversion();973 test_tools_oaicompat_json_conversion();974 test_template_output_parsers();975 std::cout << "\n[chat] All tests passed!" << '\n';976 }977 return 0;978 // } catch (const std::exception & e) {979 // std::cerr << "Error: " << e.what() << '\n';980 // return 1;981 // }982}983