Felipe97/llama-cpp-compiled
01.1k
1// Tests chat handling, including grammar genration and parsing for tool calling, for various templates.2//3// Also acts as a CLI to generate a Markdown summary of the formats of Jinja templates,4// e.g. given Minja (http://github.com/google/minja) checked out in parent dir:5//6// cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null7//8#include "../src/llama-grammar.h"9#include "../src/unicode.h"10#include "../tools/server/server-chat.h"11#include "chat-auto-parser.h"12#include "chat.h"13#include "common.h"14#include "ggml.h"15#include "log.h"16 17#include <algorithm>18#include <exception>19#include <fstream>20#include <functional>21#include <iostream>22#include "json.h"23#include <set>24#include <stdexcept>25#include <string>26 27using json = common_json;28 29static std::ostream & operator<<(std::ostream & os, const common_chat_msg_diff & diff) {30 os << "{ content_delta: " << diff.content_delta << "; ";31 os << "reasoning_content_delta: " << diff.reasoning_content_delta << "; ";32 if (diff.tool_call_index != std::string::npos) {33 os << "tool_call_index: " << diff.tool_call_index << "; ";34 os << "tool_call_delta.name: " << diff.tool_call_delta.name << "; ";35 os << "tool_call_delta.id: " << diff.tool_call_delta.id << "; ";36 os << "tool_call_delta.arguments: " << diff.tool_call_delta.arguments << "; ";37 }38 os << "}";39 return os;40}41 42// operator<< for vector<common_chat_msg_diff>:43static std::ostream & operator<<(std::ostream & os, const std::vector<common_chat_msg_diff> & diffs) {44 os << "[\n";45 for (const auto & diff : diffs) {46 os << " " << diff << ",\n";47 }48 os << "]";49 return os;50}51 52static std::ostream & operator<<(std::ostream & os, const common_chat_msg & msg) {53 os << "{ role: " << msg.role << "; ";54 os << "content: " << msg.content << "; ";55 os << "content_parts: [\n";56 for (const auto & part : msg.content_parts) {57 os << " { type: " << part.type << "; text: " << part.text << " },\n";58 }59 os << "]; ";60 os << "reasoning_content: " << msg.reasoning_content << "; ";61 os << "tool_calls: [\n";62 for (const auto & tool_call : msg.tool_calls) {63 os << " { name: " << tool_call.name << "; arguments: " << tool_call.arguments << "; id: " << tool_call.id64 << " },\n";65 }66 os << "]";67 os << "}";68 return os;69}70 71template <class T> static bool equals(const T & expected, const T & actual) {72 return expected == actual;73}74 75static common_chat_msg normalize(const common_chat_msg & msg) {76 common_chat_msg normalized = msg;77 for (auto & tool_call : normalized.tool_calls) {78 try {79 tool_call.arguments = json::parse(tool_call.arguments).dump();80 } catch (const std::exception &) {81 }82 }83 return normalized;84}85 86template <> bool equals(const common_chat_msg & expected, const common_chat_msg & actual) {87 return normalize(expected) == normalize(actual);88}89 90template <class T> static void assert_equals(const T & expected, const T & actual) {91 if (!equals(expected, actual)) {92 std::ostringstream oss_expected;93 oss_expected << expected;94 std::ostringstream oss_actual;95 oss_actual << actual;96 LOG_ERR("Expected: %s\n", oss_expected.str().c_str());97 LOG_ERR("Actual: %s\n", oss_actual.str().c_str());98 common_log_flush(common_log_main());99 throw std::runtime_error("Test failed");100 }101}102 103static void assert_contains(const std::string & haystack, const std::string & needle) {104 if (haystack.find(needle) == std::string::npos) {105 LOG_ERR("Expected to contain: %s\n", needle.c_str());106 LOG_ERR("Actual: %s\n", haystack.c_str());107 common_log_flush(common_log_main());108 throw std::runtime_error("Test failed");109 }110}111 112static void assert_not_contains(const std::string & haystack, const std::string & needle) {113 if (haystack.find(needle) != std::string::npos) {114 LOG_ERR("Expected NOT to contain: %s\n", needle.c_str());115 LOG_ERR("Actual: %s\n", haystack.c_str());116 common_log_flush(common_log_main());117 throw std::runtime_error("Test failed");118 }119}120 121static void assert_ends_with(const std::string & str, const std::string & suffix) {122 if (str.size() < suffix.size() ||123 str.compare(str.size() - suffix.size(), suffix.size(), suffix) != 0) {124 LOG_ERR("Expected to end with: %s\n", suffix.c_str());125 LOG_ERR("Actual: %s\n", str.c_str());126 common_log_flush(common_log_main());127 throw std::runtime_error("Test failed");128 }129}130 131static std::string read_file(const std::string & path) {132 std::ifstream fs(path, std::ios_base::binary);133 if (!fs.is_open()) {134 fs = std::ifstream("../" + path, std::ios_base::binary);135 if (!fs.is_open()) {136 throw std::runtime_error("Failed to open file: " + path);137 }138 }139 fs.seekg(0, std::ios_base::end);140 auto size = fs.tellg();141 fs.seekg(0);142 std::string out;143 out.resize(static_cast<size_t>(size));144 fs.read(out.data(), static_cast<std::streamsize>(size));145 return out;146}147 148static common_chat_templates_ptr read_templates(const std::string & path) {149 return common_chat_templates_ptr(common_chat_templates_init(/* model= */ nullptr, read_file(path)));150}151 152static std::unique_ptr<llama_grammar> build_grammar(const std::string & grammar_str) {153 return std::unique_ptr<llama_grammar>(154 llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0));155}156 157// Helper to format a code point as a readable string158static std::string format_codepoint(uint32_t cp) {159 if (cp >= 32 && cp < 127) {160 return std::string("'") + static_cast<char>(cp) + "'";161 } else if (cp == '\n') {162 return "'\\n'";163 } else if (cp == '\r') {164 return "'\\r'";165 } else if (cp == '\t') {166 return "'\\t'";167 } else {168 return "U+" + std::to_string(cp);169 }170}171 172// Helper to format expected element from grammar stack173static std::string format_expected_element(const llama_grammar_rules & /* rules*/, const llama_grammar_element * elem) {174 if (!elem) {175 return "<end>";176 }177 178 switch (elem->type) {179 case LLAMA_GRETYPE_END:180 return "<end of rule>";181 case LLAMA_GRETYPE_ALT:182 return "<alternative>";183 case LLAMA_GRETYPE_RULE_REF:184 {185 // Find rule name - just show rule ID for now186 return "<rule-" + std::to_string(elem->value) + ">";187 }188 case LLAMA_GRETYPE_CHAR:189 {190 std::string result;191 const llama_grammar_element * pos = elem;192 bool first = true;193 194 do {195 if (!first) {196 result += " | ";197 }198 first = false;199 200 if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {201 // Range like [a-z]202 result += "[" + format_codepoint(pos->value) + "-" + format_codepoint(pos[1].value) + "]";203 pos += 2;204 } else {205 result += format_codepoint(pos->value);206 pos += 1;207 }208 } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);209 210 return result;211 }212 case LLAMA_GRETYPE_CHAR_NOT:213 {214 std::string result = "[^";215 const llama_grammar_element * pos = elem;216 bool first = true;217 218 do {219 if (!first) {220 result += " ";221 }222 first = false;223 224 if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {225 result += format_codepoint(pos->value) + "-" + format_codepoint(pos[1].value);226 pos += 2;227 } else {228 result += format_codepoint(pos->value);229 pos += 1;230 }231 } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);232 233 return result + "]";234 }235 case LLAMA_GRETYPE_CHAR_ANY:236 return "<any char>";237 case LLAMA_GRETYPE_TOKEN:238 return "<token-" + std::to_string(elem->value) + ">";239 case LLAMA_GRETYPE_TOKEN_NOT:240 return "<not-token-" + std::to_string(elem->value) + ">";241 default:242 return "<unknown>";243 }244}245 246// Get description of what the grammar expects at current position247static std::string get_expected_description(const llama_grammar_rules & rules, const llama_grammar_stacks & stacks) {248 if (stacks.empty()) {249 return "<no valid continuations>";250 }251 252 std::string result;253 std::set<std::string> seen;254 255 for (const auto & stack : stacks) {256 if (stack.empty()) {257 if (seen.insert("<end>").second) {258 if (!result.empty()) {259 result += " OR ";260 }261 result += "<end>";262 }263 continue;264 }265 266 const llama_grammar_element * elem = stack.back();267 std::string desc = format_expected_element(rules, elem);268 if (seen.insert(desc).second) {269 if (!result.empty()) {270 result += " OR ";271 }272 result += desc;273 }274 }275 276 return result;277}278 279// Result of a detailed grammar match attempt280struct grammar_match_result {281 bool success = false; // Did the string fully match the grammar?282 size_t matched_bytes = 0; // Bytes successfully matched before failure283 size_t matched_codepoints = 0; // Codepoints successfully matched before failure284 size_t total_bytes = 0; // Total bytes in input285 size_t total_codepoints = 0; // Total codepoints in input286 std::string matched_prefix; // The portion that was successfully matched287 std::string failing_char; // The character that caused failure (if any)288 std::string expected_description; // What the grammar expected at failure point289 bool incomplete = false; // True if matched all input but grammar expects more290};291 292// Detailed version of match_string that returns failure information293static grammar_match_result match_string_detailed(const std::string & input, llama_grammar * grammar) {294 grammar_match_result result;295 result.total_bytes = input.size();296 297 const auto cpts = unicode_cpts_from_utf8(input);298 result.total_codepoints = cpts.size();299 300 auto & stacks_cur = llama_grammar_get_stacks(grammar);301 const auto & rules = llama_grammar_get_rules(grammar);302 303 size_t byte_pos = 0;304 305 for (size_t i = 0; i < cpts.size(); i++) {306 const auto & cpt = cpts[i];307 308 // Get expected before accepting (for error reporting)309 std::string expected_before = get_expected_description(rules, stacks_cur);310 311 llama_grammar_accept(grammar, cpt);312 313 // Calculate byte position for this codepoint314 size_t cpt_bytes = 0;315 if (cpt < 0x80) {316 cpt_bytes = 1;317 } else if (cpt < 0x800) {318 cpt_bytes = 2;319 } else if (cpt < 0x10000) {320 cpt_bytes = 3;321 } else {322 cpt_bytes = 4;323 }324 325 if (stacks_cur.empty()) {326 // Grammar failed to match at this point327 result.matched_bytes = byte_pos;328 result.matched_codepoints = i;329 result.matched_prefix = input.substr(0, byte_pos);330 result.failing_char = format_codepoint(cpt);331 result.expected_description = expected_before;332 result.incomplete = false;333 return result;334 }335 336 byte_pos += cpt_bytes;337 }338 339 // All input matched - check if grammar is complete340 result.matched_bytes = input.size();341 result.matched_codepoints = cpts.size();342 result.matched_prefix = input;343 344 if (std::any_of(stacks_cur.begin(), stacks_cur.end(), [](const auto & stack) { return stack.empty(); })) {345 // An empty stack means that the grammar has been completed346 result.success = true;347 result.incomplete = false;348 } else {349 // Grammar expects more input350 result.success = false;351 result.incomplete = true;352 result.expected_description = get_expected_description(rules, stacks_cur);353 }354 355 return result;356}357 358// TODO: extract to common helper (copied from test-grammar-integration.cpp)359static bool match_string(const std::string & input, llama_grammar * grammar) {360 const auto cpts = unicode_cpts_from_utf8(input);361 362 auto & stacks_cur = llama_grammar_get_stacks(grammar);363 364 for (const auto & cpt : cpts) {365 llama_grammar_accept(grammar, cpt);366 367 if (stacks_cur.empty()) {368 // no stacks means that the grammar failed to match at this point369 return false;370 }371 }372 373 if (std::any_of(stacks_cur.begin(), stacks_cur.end(), [](const auto & stack) { return stack.empty(); })) {374 // An empty stack means that the grammar has been completed375 return true;376 }377 378 return false;379}380 381static std::string renormalize_json(const std::string & json_str) {382 try {383 auto json_obj = json::parse(json_str);384 return json_obj.dump();385 } catch (const std::exception & e) {386 return ""; // ignore parial JSON contents for comparison purposes387 }388}389 390static void assert_msg_equals(const common_chat_msg & expected,391 const common_chat_msg & actual,392 bool ignore_whitespace_differences = false) {393 assert_equals(expected.role, actual.role);394 if (ignore_whitespace_differences) {395 assert_equals(string_strip(expected.content), string_strip(actual.content));396 } else {397 assert_equals(expected.content, actual.content);398 }399 assert_equals(expected.content_parts.size(), actual.content_parts.size());400 for (size_t i = 0; i < expected.content_parts.size(); i++) {401 const auto & expected_part = expected.content_parts[i];402 const auto & actual_part = actual.content_parts[i];403 assert_equals(expected_part.type, actual_part.type);404 if (ignore_whitespace_differences) {405 assert_equals(string_strip(expected_part.text), string_strip(actual_part.text));406 } else {407 assert_equals(expected_part.text, actual_part.text);408 }409 }410 if (ignore_whitespace_differences) {411 assert_equals(string_strip(expected.reasoning_content), string_strip(actual.reasoning_content));412 } else {413 assert_equals(expected.reasoning_content, actual.reasoning_content);414 }415 assert_equals(expected.tool_calls.size(), actual.tool_calls.size());416 for (size_t i = 0; i < expected.tool_calls.size(); i++) {417 const auto & expected_tool_call = expected.tool_calls[i];418 const auto & actual_tool_call = actual.tool_calls[i];419 assert_equals(expected_tool_call.name, actual_tool_call.name);420 assert_equals(renormalize_json(expected_tool_call.arguments), renormalize_json(actual_tool_call.arguments));421 assert_equals(expected_tool_call.id, actual_tool_call.id);422 }423}424 425static common_chat_tool special_function_tool{426 /* .name = */ "special_function",427 /* .description = */ "I'm special",428 /* .parameters = */ R"({429 "type": "object",430 "properties": {431 "arg1": {432 "type": "integer",433 "description": "The arg."434 }435 },436 "required": ["arg1"]437 })",438};439static common_chat_tool special_function_tool_with_optional_param{440 /* .name = */ "special_function_with_opt",441 /* .description = */ "I'm special but have optional stuff",442 /* .parameters = */ R"({443 "type": "object",444 "properties": {445 "arg1": {446 "type": "integer",447 "description": "The arg."448 },449 "arg2": {450 "type": "integer",451 "description": "The optional arg."452 }453 },454 "required": ["arg1"]455 })",456};457 458static common_chat_tool empty_args_tool{459 /* .name = */ "empty_args",460 /* .description = */ "A tool that takes no arguments",461 /* .parameters = */ R"({462 "type": "object",463 "properties": {}464 })",465};466 467static common_chat_tool empty_args_tool_no_properties{468 /* .name = */ "empty_args_no_props",469 /* .description = */ "A tool that takes no arguments and has no properties",470 /* .parameters = */ R"({471 "type": "object"472 })",473};474 475static common_chat_tool empty_args_tool_no_schema{476 /* .name = */ "empty_args_no_schema",477 /* .description = */ "A tool that takes no arguments and has no parameters schema",478 /* .parameters = */ "{}",479};480 481static common_chat_tool python_tool{482 /* .name = */ "python",483 /* .description = */ "an ipython interpreter",484 /* .parameters = */ R"({485 "type": "object",486 "properties": {487 "code": {488 "type": "string",489 "description": "Python code to execute."490 }491 },492 "required": ["code"]493 })",494};495 496static common_chat_tool html_tool{497 /* .name = */ "html",498 /* .description = */ "an html validator",499 /* .parameters = */ R"({500 "type": "object",501 "properties": {502 "markup": {503 "type": "string",504 "description": "HTML markup to validate."505 }506 },507 "required": ["markup"]508 })",509};510 511static common_chat_tool get_time_tool{512 /* .name = */ "get_time",513 /* .description = */ "Get the current time in a city",514 /* .parameters = */ R"({515 "type": "object",516 "properties": {517 "city": {518 "type": "string",519 "description": "City name"520 }521 },522 "required": ["city"]523 })",524};525 526static common_chat_tool get_weather_tool{527 /* .name = */ "get_weather",528 /* .description = */ "Get the current weather in a city",529 /* .parameters = */ R"({530 "type": "object",531 "properties": {532 "city": {533 "type": "string",534 "description": "City name"535 }536 },537 "required": ["city"]538 })",539};540 541static common_chat_tool todo_list{542 /* .name = */ "todo_list",543 /* .description = */ "Create or update the todo list",544 /* .parameters = */ R"({545 "type": "object",546 "properties": {547 "todos": {548 "type": "array",549 "description": "List of TODO list items"550 }551 },552 "required": ["todos"]553 })",554};555 556static common_chat_tool edit_tool{557 /* .name = */ "edit",558 /* .description = */ "Edit file",559 /* .parameters = */ R"({560 "type": "object",561 "properties": {562 "filename": {563 "type": "string",564 "description": "Path of file to edit"565 },566 "oldString": {567 "type": "string",568 "description": "String to replace"569 },570 "newString": {571 "type": "string",572 "description": "New (replacement) value"573 }574 },575 "required": ["filename", "oldString", "newString"]576 })",577};578 579static common_chat_tool manage_todo_list_tool{580 /* .name = */ "manage_todo_list",581 /* .description = */ "Create or update the todo list",582 /* .parameters = */ R"({583 "type": "object",584 "properties": {585 "todos": {586 "type": "array",587 "description": "List of TODO list items"588 }589 },590 "required": ["todos"]591 })",592};593 594static common_chat_tool run_in_terminal_tool{595 /* .name = */ "run_in_terminal",596 /* .description = */ "Run a shell command.",597 /* .parameters = */ R"({598 "type": "object",599 "properties": {600 "command": {601 "type": "string",602 "description": "Shell command to run"603 }604 },605 "required": ["command"]606 })",607};608 609static common_chat_tool magic_tool{610 /* .name = */ "magic",611 /* .description = */ "Magic tool that takes a hash",612 /* .parameters = */ R"({613 "type": "object",614 "properties": {615 "name": {616 "type": "string"617 },618 "ref": {619 "type": "string"620 }621 },622 "required": ["name", "ref"]623 })",624};625 626static common_chat_tool magic_int_tool{627 /* .name = */ "magic_int",628 /* .description = */ "Magic tool that takes a hash",629 /* .parameters = */ R"({630 "type": "object",631 "properties": {632 "ref": {633 "type": "integer"634 },635 "name": {636 "type": "string"637 }638 },639 "required": ["ref"]640 })",641};642 643static common_chat_tool amount_tool{644 /* .name = */ "amount",645 /* .description = */ "Amount converter",646 /* .parameters = */ R"({647 "type": "object",648 "properties": {649 "orig": {650 "type": "number"651 }652 },653 "required": ["orig"]654 })",655};656 657static common_chat_tool toggle_tool{658 /* .name = */ "toggle",659 /* .description = */ "Toggle a feature",660 /* .parameters = */ R"({661 "type": "object",662 "properties": {663 "enabled": {664 "type": "boolean",665 "description": "Whether to enable the feature"666 }667 },668 "required": ["enabled"]669 })",670};671 672static common_chat_tool nullable_tool{673 /* .name = */ "set_nullable",674 /* .description = */ "Set a nullable value",675 /* .parameters = */ R"({676 "type": "object",677 "properties": {678 "value": {679 "type": "null",680 "description": "A null value"681 }682 },683 "required": ["value"]684 })",685};686 687static common_chat_tool config_tool{688 /* .name = */ "set_config",689 /* .description = */ "Set configuration",690 /* .parameters = */ R"({691 "type": "object",692 "properties": {693 "config": {694 "type": "object",695 "description": "Configuration dict"696 }697 },698 "required": ["config"]699 })",700};701 702static common_chat_tool calendar_create_event_tool{703 /* .name = */ "Calendar.create_event",704 /* .description = */ "Create a calendar event",705 /* .parameters = */ R"({706 "type": "object",707 "properties": {708 "title": { "type": "string" },709 "participants": { "type": "array", "items": { "type": "string" } },710 "metadata": { "type": "object" }711 },712 "required": ["title", "participants", "metadata"]713 })",714};715 716static common_chat_tool imaginary_number_tool{717 /* .name = */ "imaginary_number",718 /* .description = */ "Imaginary number converter",719 /* .parameters = */ R"({720 "type": "object",721 "properties": {722 "number": {723 "type": "object",724 "properties": {725 "real": {726 "type": "number"727 },728 "imaginary": {729 "type": "number"730 }731 },732 "required": ["real", "imaginary"]733 }734 },735 "required": ["number"]736 })",737};738 739static common_chat_tool nested_args_tool{740 /* .name = */ "nested_args",741 /* .description = */ "Tool with nested array arguments",742 /* .parameters = */ R"({743 "type": "object",744 "properties": {745 "tags": {746 "type": "array",747 "items": { "type": "string" }748 },749 "entries": {750 "type": "array",751 "items": {752 "type": "object",753 "properties": {754 "id": { "type": "integer" },755 "label": { "type": "string" }756 },757 "required": ["id", "label"]758 }759 }760 },761 "required": ["tags", "entries"]762 })",763};764 765static common_chat_tool union_args_tool{766 /* .name = */ "union_args",767 /* .description = */ "Tool with union arguments",768 /* .parameters = */ R"({769 "type": "object",770 "properties": {771 "filter": {772 "anyOf": [773 { "type": "array", "items": { "type": "string" } },774 {775 "type": "object",776 "properties": {777 "field": { "type": "string" },778 "op": { "type": "string" }779 },780 "required": ["field", "op"]781 }782 ]783 },784 "label": {785 "oneOf": [786 { "type": "string" },787 { "type": "object", "properties": { "text": { "type": "string" } } }788 ]789 },790 "limit": {791 "oneOf": [792 { "type": "integer" },793 {794 "type": "object",795 "properties": { "max": { "type": "integer" } },796 "required": ["max"]797 }798 ]799 }800 }801 })",802};803 804static common_chat_tool nullable_string_tool{805 /* .name = */ "set_nullable_str",806 /* .description = */ "Set a nullable string value",807 /* .parameters = */ R"({808 "type": "object",809 "properties": {810 "name": {811 "type": ["string", "null"],812 "description": "A nullable string"813 }814 },815 "required": ["name"]816 })",817};818 819static common_chat_tool nullable_string_null_first_tool{820 /* .name = */ "set_nullable_str_nf",821 /* .description = */ "Set a nullable string value with null first in type array",822 /* .parameters = */ R"({823 "type": "object",824 "properties": {825 "name": {826 "type": ["null", "string"],827 "description": "A nullable string with null first"828 }829 },830 "required": ["name"]831 })",832};833 834static common_chat_tool nullable_int_tool{835 /* .name = */ "set_nullable_int",836 /* .description = */ "Set a nullable integer value",837 /* .parameters = */ R"({838 "type": "object",839 "properties": {840 "count": {841 "type": ["integer", "null"],842 "description": "A nullable integer"843 }844 },845 "required": ["count"]846 })",847};848 849static common_chat_tool string_union_tool{850 /* .name = */ "set_union",851 /* .description = */ "Set values whose types are unions with string",852 /* .parameters = */ R"({853 "type": "object",854 "properties": {855 "value": {856 "type": ["string", "object"],857 "description": "A string or object value"858 },859 "amount": {860 "type": ["string", "integer"],861 "description": "A string or integer value"862 }863 },864 "required": ["value", "amount"]865 })",866};867 868static common_chat_tool enum_no_type_tool{869 /* .name = */ "set_unit",870 /* .description = */ "Set a temperature unit",871 /* .parameters = */ R"({872 "type": "object",873 "properties": {874 "unit": {875 "enum": ["celsius", "fahrenheit"],876 "description": "Temperature unit"877 }878 },879 "required": ["unit"]880 })",881};882 883static common_chat_tool string_param_tool{884 /* .name = */ "string_param",885 /* .description = */ "Tool with string parameter for testing",886 /* .parameters = */ R"({887 "type": "object",888 "properties": {889 "text": {890 "type": "string",891 "description": "A text parameter"892 }893 },894 "required": []895 })",896};897 898static common_chat_tool quoted_unquoted_tool{899 /* .name = */ "quoted_unquoted",900 /* .description = */ "Tool with two string parameters, one for quoted string, one for unquoted",901 /* .parameters = */ R"({902 "type": "object",903 "properties": {904 "quoted": {905 "type": "string",906 "description": "Quoted value"907 },908 "unquoted": {909 "type": "string",910 "description": "Unquoted value"911 }912 },913 "required": ["quoted", "unquoted"]914 })",915};916 917 918static common_chat_tool tool_2req_4opt{919 /* .name = */ "tool_2req_4opt",920 /* .description = */ "Tool with 2 required and 4 optional params",921 /* .parameters = */ R"({922 "type": "object",923 "properties": {924 "req1": { "type": "string", "description": "Required string" },925 "req2": { "type": "integer", "description": "Required int" },926 "opt1": { "type": "string", "description": "Optional string 1" },927 "opt2": { "type": "integer", "description": "Optional int 1" },928 "opt3": { "type": "string", "description": "Optional string 2" },929 "opt4": { "type": "integer", "description": "Optional int 2" }930 },931 "required": ["req1", "req2"]932 })",933};934 935static common_chat_tool tool_2req_5opt{936 /* .name = */ "tool_2req_5opt",937 /* .description = */ "Tool with 2 required and 5 optional params",938 /* .parameters = */ R"({939 "type": "object",940 "properties": {941 "req1": { "type": "string", "description": "Required string" },942 "req2": { "type": "integer", "description": "Required int" },943 "opt1": { "type": "string", "description": "Optional string 1" },944 "opt2": { "type": "integer", "description": "Optional int 1" },945 "opt3": { "type": "string", "description": "Optional string 2" },946 "opt4": { "type": "integer", "description": "Optional int 2" },947 "opt5": { "type": "string", "description": "Optional string 3" }948 },949 "required": ["req1", "req2"]950 })",951};952 953static std::vector<common_chat_tool> tools{ special_function_tool, special_function_tool_with_optional_param,954 python_tool, html_tool, todo_list };955 956const common_chat_msg message_user{957 "user",958 "Hey there!",959 /* .content_parts = */ {},960 /* .tool_calls = */ {},961 /* .reasoning_content = */ "",962 /* .tool_name = */ "",963 /* .tool_call_id = */ "",964};965 966const common_chat_msg message_user_parts{967 "user",968 /* .content = */ "",969 /* .content_parts = */970 {971 { "text", "Hey" },972 { "text", "there" },973 },974 /* .tool_calls = */975 { },976 /* .reasoning_content = */977 "",978 /* .tool_name = */ "",979 /* .tool_call_id = */ "",980};981 982static common_chat_msg simple_assist_msg(const std::string & content,983 const std::string & reasoning_content = "",984 const std::string & tool_name = "",985 const std::string & arguments = "",986 const std::string & id = "") {987 common_chat_msg msg;988 msg.role = "assistant";989 msg.content = content;990 msg.reasoning_content = reasoning_content;991 if (!tool_name.empty() || !id.empty()) {992 msg.tool_calls.push_back({ tool_name, arguments, id });993 }994 return msg;995}996 997static common_chat_msg message_with_tool_calls(const std::string & tool_name, const std::string & arguments) {998 return simple_assist_msg("", "", tool_name, arguments);999}1000 1001static common_chat_msg message_with_tool_calls_and_reasoning(const std::string & tool_name,1002 const std::string & arguments,1003 const std::string & reasoning) {1004 return simple_assist_msg("", reasoning, tool_name, arguments);1005}1006 1007static common_chat_msg message_with_reasoning_content_and_multiple_tool_calls(1008 const std::string & reasoning,1009 const std::string & content,1010 const std::vector<std::pair<std::string, std::string>> & tool_calls) {1011 common_chat_msg msg;1012 msg.role = "assistant";1013 msg.content = content;1014 msg.reasoning_content = reasoning;1015 for (const auto & [name, args] : tool_calls) {1016 msg.tool_calls.push_back({ name, args, "" });1017 }1018 return msg;1019}1020 1021static common_chat_msg message_with_content_and_tool_call(const std::string & content,1022 const std::string & tool_name,1023 const std::string & arguments) {1024 return simple_assist_msg(content, "", tool_name, arguments);1025}1026 1027static common_chat_msg message_with_reasoning_and_tool_call(const std::string & reasoning,1028 const std::string & tool_name,1029 const std::string & arguments) {1030 return simple_assist_msg("", reasoning, tool_name, arguments);1031}1032 1033const common_chat_msg message_assist = simple_assist_msg("Hello, world!\nWhat's up?");1034const common_chat_msg message_assist_empty = simple_assist_msg("");1035const common_chat_msg message_assist_thoughts_unparsed_deepseek =1036 simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?");1037const common_chat_msg message_assist_thoughts_unparsed_md =1038 simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}```");1039const common_chat_msg message_assist_thoughts_unparsed_md_partial =1040 simple_assist_msg("<think>I'm\nthinking</think>Hello, world!\nWhat's up?\n```json\n{}");1041 1042const common_chat_msg message_assist_thoughts_unparsed_r7b =1043 simple_assist_msg("<|START_THINKING|>I'm\nthinking<|END_THINKING|>Hello, world!\nWhat's up?");1044const common_chat_msg message_assist_thoughts_unparsed_magistral =1045 simple_assist_msg("[THINK]raisonnement[/THINK]Réponse");1046const common_chat_msg message_assist_thoughts = simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking");1047const common_chat_msg message_assist_thoughts_unopened_unparsed =1048 simple_assist_msg("I'm\nthinking</think>Hello, world!\nWhat's up?");1049const common_chat_msg message_assist_thoughts_no_content = simple_assist_msg("", "I'm\nthinking");1050const common_chat_msg message_assist_call = simple_assist_msg("", "", "special_function", "{\"arg1\": 1}");1051const common_chat_msg message_assist_call_noopt =1052 simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1}");1053const common_chat_msg message_assist_call_withopt =1054 simple_assist_msg("", "", "special_function_with_opt", "{\"arg1\": 1, \"arg2\": 2}");1055const common_chat_msg message_assist_call_content =1056 simple_assist_msg("Hello, world!\nWhat's up?", "", "special_function", "{\"arg1\":1}");1057const common_chat_msg message_assist_call_empty_args = simple_assist_msg("", "", "special_function");1058const common_chat_msg message_assist_call_cutoff_args = simple_assist_msg("", "", "special_function", "{\"arg");1059const common_chat_msg message_assist_call_thoughts =1060 simple_assist_msg("", "I'm\nthinking", "special_function", "{\"arg1\":1}");1061const common_chat_msg message_assist_call_thoughts_unparsed =1062 simple_assist_msg("<think>I'm\nthinking</think>\n\n", "", "special_function", "{\"arg1\": 1}");1063const common_chat_msg message_assist_call_thoughts_content =1064 simple_assist_msg("Hello, world!\nWhat's up?", "I'm\nthinking", "special_function", "{\"arg1\": 1}");1065const common_chat_msg message_assist_call_id =1066 simple_assist_msg("", "", "special_function", "{\"arg1\":1}", /* .id = */ "123456789");1067const common_chat_msg message_assist_call_idx =1068 simple_assist_msg("", "", "special_function", "{\"arg1\":1}", /* .id = */ "0");1069const common_chat_msg message_assist_thoughts_call_idx =1070 simple_assist_msg("", "I'm\nthinking", "special_function", "{\"arg1\": 1}", /* id = */ "0");1071const common_chat_msg message_assist_thoughts_partial_call =1072 simple_assist_msg("", "I'm\nthinking", "special_function", "", /* id = */ "0");1073const common_chat_msg message_assist_call_python = simple_assist_msg("", "", "python", "{\"code\":\"print('hey')\"}");1074const common_chat_msg message_assist_call_python_lines =1075 simple_assist_msg("", "", "python", "{\"code\":\"# This is a program:\\nprint('hey')\"}");1076const common_chat_msg message_assist_call_python_lines_unclosed =1077 simple_assist_msg("", "", "python", "{\"code\":\"# This is a program:\\nprint('hey')");1078const common_chat_msg message_assist_json_content =1079 simple_assist_msg("{\n \"response\": \"Hello, world!\\nWhat's up?\"\n}");1080const common_chat_msg message_assist_prefill_content = simple_assist_msg("Hello, ", "I'm thinking");1081const common_chat_msg message_assist_prefill_reasoning = simple_assist_msg("", "I'm");1082 1083// Use for PEG parser implementations1084struct peg_test_case {1085 common_chat_templates_inputs params;1086 std::string input;1087 common_chat_msg expect;1088 bool is_partial = false;1089 bool expect_reconstruction = false;1090};1091 1092struct make_peg_parser {1093 common_chat_params params_;1094 common_peg_arena arena_;1095 bool detailed_debug_;1096 1097 make_peg_parser(common_chat_templates * tmpls,1098 const common_chat_templates_inputs & inputs,1099 bool detailed_debug = false) {1100 detailed_debug_ = detailed_debug;1101 params_ = common_chat_templates_apply(tmpls, inputs);1102 arena_.load(params_.parser);1103 }1104 1105 common_chat_msg parse(const std::string & msg, bool is_partial) const {1106 common_chat_parser_params parser_params(params_);1107 parser_params.debug = detailed_debug_;1108 return common_chat_peg_parse(arena_, msg, is_partial, parser_params);1109 }1110};1111 1112// Global template filter for --template flag1113static std::string g_template_filter;1114 1115// When true, run reconstruction test on every non-partial test and report results1116static bool g_force_reconstruction_test = false;1117 1118static void test_peg_parser(common_chat_templates * tmpls,1119 const std::function<void(peg_test_case &)> & init,1120 bool detailed_debug) {1121 // UTF-8-safe truncation helper (same as in test_parser_with_streaming)1122 constexpr auto utf8_truncate_safe_len = [](const std::string_view s) -> size_t {1123 auto len = s.size();1124 if (len == 0) {1125 return 0;1126 }1127 auto i = len;1128 for (size_t back = 0; back < 4 && i > 0; ++back) {1129 --i;1130 unsigned char c = s[i];1131 if ((c & 0x80) == 0) {1132 return len;1133 }1134 if ((c & 0xC0) == 0xC0) {1135 size_t expected_len = 0;1136 if ((c & 0xE0) == 0xC0) {1137 expected_len = 2;1138 } else if ((c & 0xF0) == 0xE0) {1139 expected_len = 3;1140 } else if ((c & 0xF8) == 0xF0) {1141 expected_len = 4;1142 } else {1143 return i;1144 }1145 if (len - i >= expected_len) {1146 return len;1147 }1148 return i;1149 }1150 }1151 return len - std::min(len, size_t(3));1152 };1153 1154 peg_test_case tc;1155 init(tc);1156 if (tc.params.messages.empty()) {1157 tc.params.messages = { message_user };1158 }1159 if (tc.expect.role.empty()) {1160 tc.expect.role = "assistant";1161 }1162 1163 auto parser = make_peg_parser(tmpls, tc.params, detailed_debug);1164 if (detailed_debug) {1165 LOG_DBG("Using parser: \n%s\n", parser.arena_.dump(parser.arena_.root()).c_str());1166 LOG_DBG("Generation prompt: '%s'\n", parser.params_.generation_prompt.c_str());1167 }1168 1169 common_chat_msg msg_accum;1170 common_chat_msg msg_prev;1171 msg_accum.role = msg_prev.role = "assistant";1172 1173 for (size_t i = 1; i <= tc.input.size(); ++i) {1174 auto is_partial = i < tc.input.size() || tc.is_partial;1175 // Use UTF-8 safe truncation to avoid corrupting multi-byte characters1176 size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i));1177 std::string prefix = tc.input.substr(0, safe_len);1178 common_chat_msg msg_current = parser.parse(prefix, is_partial);1179 1180 for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) {1181 if (!diff.reasoning_content_delta.empty()) {1182 msg_accum.reasoning_content += diff.reasoning_content_delta;1183 }1184 if (!diff.content_delta.empty()) {1185 msg_accum.content += diff.content_delta;1186 }1187 if (diff.tool_call_index != std::string::npos) {1188 // During partial parsing, a new tool call may appear with empty name initially1189 // The name gets filled in as more input is parsed1190 while (msg_accum.tool_calls.size() <= diff.tool_call_index) {1191 msg_accum.tool_calls.push_back({ "", "", "" });1192 }1193 // Always update name and id from diff (may change during incremental parsing), but only if the delta1194 // actually contains them1195 if (!diff.tool_call_delta.name.empty()) {1196 msg_accum.tool_calls[diff.tool_call_index].name = diff.tool_call_delta.name;1197 }1198 if (!diff.tool_call_delta.id.empty()) {1199 msg_accum.tool_calls[diff.tool_call_index].id = diff.tool_call_delta.id;1200 }