Felipe97/llama-cpp-compiled
01.1k
1#include "chat-auto-parser-helpers.h"2#include "chat-auto-parser.h"3#include "chat-peg-parser.h"4#include "chat.h"5#include "gguf.h"6#include "jinja/runtime.h"7#include "log.h"8#include "peg-parser.h"9#include "testing.h"10 11#include <cstdlib>12#include <filesystem>13#include <fstream>14#include <iostream>15#include <iterator>16#include <optional>17#include <sstream>18#include <string>19 20using namespace autoparser;21 22static void test_calculate_diff_split_basic(testing & t);23static void test_calculate_diff_split_identical(testing & t);24static void test_calculate_diff_split_common_prefix(testing & t);25static void test_calculate_diff_split_common_suffix(testing & t);26static void test_calculate_diff_split_common_both(testing & t);27static void test_calculate_diff_split_empty_cases(testing & t);28static void test_calculate_diff_split_no_common(testing & t);29static void test_calculate_diff_split_single_char(testing & t);30static void test_calculate_diff_split_overlaps(testing & t);31static void test_calculate_diff_split_tag_boundaries(testing & t);32static void test_calculate_diff_split_generation_prompt(testing & t);33static void test_calculate_diff_split(testing & t);34 35static void test_until_common_prefix_basic(testing & t);36static void test_until_common_prefix(testing & t);37 38static void test_after_common_suffix_basic(testing & t);39static void test_after_common_suffix(testing & t);40 41static void test_analyze_tool_call_pure_json(testing & t);42static void test_analyze_tool_call_function_name_markers(testing & t);43static void test_analyze_tool_call_full_markers(testing & t);44static void test_analyze_tool_call_edge_cases(testing & t);45 46static void test_compare_variants_basic(testing & t);47static void test_compare_variants_messages_modifier(testing & t);48static void test_compare_variants_tools_modifier(testing & t);49static void test_compare_variants_both_modifiers(testing & t);50static void test_compare_variants_template_failure(testing & t);51static void test_compare_variants_identity(testing & t);52static void test_compare_variants(testing & t);53 54// Seed-OSS template tool calling analysis tests55static void test_seed_oss_tool_analysis(testing & t);56static void test_seed_oss_tool_presence(testing & t);57static void test_seed_oss_call_count(testing & t);58static void test_seed_oss_function_names(testing & t);59static void test_seed_oss_argument_count(testing & t);60static void test_seed_oss_args_presence(testing & t);61static void test_seed_oss_tool_with_reasoning(testing & t);62 63// Nemotron template analysis tests64static void test_nemotron_analysis(testing & t);65static void test_nemotron_reasoning_detection(testing & t);66static void test_nemotron_tool_format(testing & t);67static void test_laguna_analysis(testing & t);68static void test_laguna_reasoning_detection(testing & t);69static void test_laguna_tool_format(testing & t);70static void test_laguna_s_analysis(testing & t);71static void test_laguna_s_reasoning_detection(testing & t);72static void test_laguna_s_tool_format(testing & t);73static void test_laguna_s_preserve_reasoning(testing & t);74static void test_laguna_xs2_analysis(testing & t);75static void test_laguna_xs2_reasoning_detection(testing & t);76static void test_laguna_xs2_tool_format(testing & t);77 78// CohereForAI template analysis tests79static void test_cohere_reasoning_detection(testing & t);80static void test_cohere_analysis(testing & t);81 82// SmolLM3 template analysis tests83static void test_smollm3_analysis(testing & t);84 85// Marker separation86static void test_marker_separation(testing & t);87 88// standard_json_tools format tests89static void test_standard_json_tools_formats(testing & t);90static void test_standard_json_tools_openai(testing & t);91static void test_standard_json_tools_cohere(testing & t);92static void test_standard_json_tools_function_key(testing & t);93 94// normalize_quotes_to_json tests95static void test_normalize_quotes_to_json(testing & t);96static void test_normalize_quotes_with_embedded_quotes(testing & t);97 98// TAG_WITH_TAGGED argument parsing tests99static void test_tagged_args_with_embedded_quotes(testing & t);100static void test_bailing_v3_tool_format(testing & t);101 102static void test_role_markers_all_templates(testing & t);103 104static json build_tools_definition();105 106//107// debug mode: analyze a single template and dump the generated parser and grammar108//109 110enum class output_mode {111 ANALYSIS, // Only output analysis results (default)112 TEMPLATE, // Only output rendered template113 BOTH // Output both114};115 116enum class input_message_type {117 NONE, // Don't render any message scenarios (only analysis)118 CONTENT_ONLY, // Simple assistant message with content119 REASONING_CONTENT, // Message with reasoning_content + content120 TOOL_CALL_ONLY, // Message with tool_calls only121 CONTENT_TOOL_CALL, // Message with content + tool_calls122 REASONING_TOOL_CALL, // Message with reasoning_content + tool_calls123 CONTENT_FAKE_TOOL_CALL, // Message with content but no actual tool_calls (for testing)124 ALL // Render all scenarios125};126 127struct debug_options {128 std::string template_path;129 bool with_tools = true;130 bool generation_prompt = true;131 bool enable_reasoning = true;132 bool debug_jinja = false;133 bool force_tool_call = false;134 bool parallel_tool_calls = true;135 output_mode mode = output_mode::BOTH;136 input_message_type input_message = input_message_type::NONE;137};138 139static std::string read_file(const std::string & path) {140 std::ifstream fin(path, std::ios::binary);141 if (!fin.is_open()) {142 throw std::runtime_error("Could not open file: " + path);143 }144 std::ostringstream buf;145 buf << fin.rdbuf();146 return buf.str();147}148 149static std::string read_gguf_chat_template(const std::string & path) {150 struct gguf_init_params params = { /*no_alloc =*/true, // We only need metadata, not tensor data151 /*ctx=*/nullptr };152 153 struct gguf_context * ctx = gguf_init_from_file(path.c_str(), params);154 if (ctx == nullptr) {155 throw std::runtime_error("Could not open GGUF file: " + path);156 }157 158 const char * key = "tokenizer.chat_template";159 int64_t key_id = gguf_find_key(ctx, key);160 161 if (key_id == -1) {162 gguf_free(ctx);163 throw std::runtime_error("GGUF file does not contain chat template key: " + std::string(key));164 }165 166 const char * template_str = gguf_get_val_str(ctx, key_id);167 if (template_str == nullptr) {168 gguf_free(ctx);169 throw std::runtime_error("GGUF file contains chat template key but value is null");170 }171 172 std::string result = template_str;173 gguf_free(ctx);174 return result;175}176 177static void print_usage(const char * program_name) {178 LOG_ERR("Test the chat template auto-parser; also usable as a debug tool that shows the generated PEG parser, GBNF grammar and triggers for a given template.\n");179 LOG_ERR("\nUsage: %s [filter_regex] run the automated tests (default)\n", program_name);180 LOG_ERR(" %s <template_or_gguf_path> [options] debug a single template\n", program_name);181 LOG_ERR("\nDebug mode options:\n");182 LOG_ERR(" --no-tools Disable tool definitions\n");183 LOG_ERR(" --force-tool-call Set tool calls to forced\n");184 LOG_ERR(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n");185 LOG_ERR(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n");186 LOG_ERR(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n");187 LOG_ERR(" --output=MODE Output mode: analysis, template, both (default: both)\n");188 LOG_ERR(" --debug-jinja Enable Jinja fine-grained debug\n");189 LOG_ERR(" --input-message=TYPE Message type to render:\n");190 LOG_ERR(" content_only, reasoning_content, tool_call_only,\n");191 LOG_ERR(" content_tool_call, reasoning_tool_call,\n");192 LOG_ERR(" content_fake_tool_call, all\n");193 LOG_ERR("\nExamples:\n");194 LOG_ERR(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name);195 LOG_ERR(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name);196}197 198static bool parse_bool_option(const std::string & value) {199 return value == "1" || value == "true" || value == "yes";200}201 202static bool parse_debug_options(int argc, char ** argv, debug_options & opts) {203 opts.template_path = argv[1];204 205 for (int i = 2; i < argc; ++i) {206 std::string arg = argv[i];207 208 if (arg == "--force-tool-call") {209 opts.force_tool_call = true;210 } else if (arg == "--debug-jinja") {211 opts.debug_jinja = true;212 } else if (arg == "--no-tools") {213 opts.with_tools = false;214 } else if (arg.rfind("--parallel-tool-calls=", 0) == 0) {215 opts.parallel_tool_calls = parse_bool_option(arg.substr(22));216 } else if (arg.rfind("--generation-prompt=", 0) == 0) {217 opts.generation_prompt = parse_bool_option(arg.substr(20));218 } else if (arg.rfind("--enable-reasoning=", 0) == 0) {219 opts.enable_reasoning = parse_bool_option(arg.substr(19));220 } else if (arg.rfind("--output=", 0) == 0) {221 std::string mode = arg.substr(9);222 if (mode == "analysis") {223 opts.mode = output_mode::ANALYSIS;224 } else if (mode == "template") {225 opts.mode = output_mode::TEMPLATE;226 } else if (mode == "both") {227 opts.mode = output_mode::BOTH;228 } else {229 LOG_ERR("Unknown output mode: %s\n", mode.c_str());230 return false;231 }232 } else if (arg.rfind("--input-message=", 0) == 0) {233 std::string type = arg.substr(16);234 if (type == "content_only") {235 opts.input_message = input_message_type::CONTENT_ONLY;236 } else if (type == "reasoning_content") {237 opts.input_message = input_message_type::REASONING_CONTENT;238 } else if (type == "tool_call_only") {239 opts.input_message = input_message_type::TOOL_CALL_ONLY;240 } else if (type == "content_tool_call") {241 opts.input_message = input_message_type::CONTENT_TOOL_CALL;242 } else if (type == "reasoning_tool_call") {243 opts.input_message = input_message_type::REASONING_TOOL_CALL;244 } else if (type == "content_fake_tool_call") {245 opts.input_message = input_message_type::CONTENT_FAKE_TOOL_CALL;246 } else if (type == "all") {247 opts.input_message = input_message_type::ALL;248 } else {249 LOG_ERR("Unknown input message type: %s\n", type.c_str());250 return false;251 }252 } else {253 LOG_ERR("Unknown option: %s\n", arg.c_str());254 print_usage(argv[0]);255 return false;256 }257 }258 259 return true;260}261 262static json build_debug_user_message() {263 return json{264 { "role", "user" },265 { "content", "Hello, please help me with a task." }266 };267}268 269static json build_content_only_message() {270 return json{271 { "role", "assistant" },272 { "content", "Hello! I'm here to help you with your task." }273 };274}275 276static json build_reasoning_content_message() {277 return json{278 { "role", "assistant" },279 { "content", "Hello! I'm here to help you with your task." },280 { "reasoning_content", "The user is greeting me and asking for help. I should respond politely." }281 };282}283 284static json build_tool_call_only_message() {285 return json{286 { "role", "assistant" },287 { "content", nullptr },288 { "tool_calls",289 json::array({ json{290 { "type", "function" },291 { "function", json{ { "name", "test_function_name" },292 { "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } },293 { "id", "123456789" } } }) }294 };295}296 297static json build_content_tool_call_message() {298 return json{299 { "role", "assistant" },300 { "content", "I'll help you by calling a function." },301 { "tool_calls",302 json::array({ json{303 { "type", "function" },304 { "function",305 json{ { "name", "test_function_name" },306 { "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) }307 };308}309 310static json build_reasoning_tool_call_message() {311 return json{312 { "role", "assistant" },313 { "content", nullptr },314 { "reasoning_content", "I need to call a function to help with this task." },315 { "tool_calls",316 json::array({ json{317 { "type", "function" },318 { "function",319 json{ { "name", "test_function_name" },320 { "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) }321 };322}323 324static json build_content_fake_tool_call_message() {325 // This message has content but NO tool_calls field326 // It's used to test if a template renders tool definitions but not tool calls327 return json{328 { "role", "assistant" },329 { "content", "I'll help you by calling a function." }330 };331}332 333static void render_scenario(const common_chat_template & tmpl,334 const std::string & scenario_name,335 const json & messages,336 const json & tools,337 bool add_generation_prompt,338 bool enable_thinking) {339 LOG_ERR("\n=== Scenario: %s ===\n", scenario_name.c_str());340 LOG_ERR("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false",341 enable_thinking ? "true" : "false");342 343 // When add_generation_prompt is true, add a trailing user message to trigger the prompt344 json final_messages = messages;345 if (add_generation_prompt && !messages.empty() && messages.back().value("role", "") == "assistant") {346 final_messages.push_back(json{347 { "role", "user" },348 { "content", "Now please continue with another response." }349 });350 }351 352 LOG_ERR("Messages:\n%s\n", final_messages.dump(2).c_str());353 354 try {355 generation_params inputs;356 inputs.messages = final_messages;357 inputs.add_generation_prompt = add_generation_prompt;358 inputs.extra_context["enable_thinking"] = enable_thinking;359 360 if (!tools.is_null() && tools.is_array() && !tools.empty()) {361 inputs.tools = tools;362 }363 364 std::string output = common_chat_template_direct_apply(tmpl, inputs);365 366 LOG_ERR("\n--- Rendered Output ---\n");367 LOG_ERR("%s\n", output.c_str());368 LOG_ERR("--- End Output (length: %zu) ---\n", output.length());369 } catch (const std::exception & e) {370 LOG_ERR("Rendering failed: %s\n", e.what());371 }372}373 374static void render_all_scenarios(const common_chat_template & tmpl,375 const json & tools,376 bool add_generation_prompt,377 bool enable_thinking,378 input_message_type message_type) {379 json user_msg = build_debug_user_message();380 381 auto render_if = [&](input_message_type type, const std::string & name, const json & assistant_msg) {382 if (message_type == input_message_type::ALL || message_type == type) {383 json messages = json::array({ user_msg, assistant_msg });384 render_scenario(tmpl, name, messages, tools, add_generation_prompt, enable_thinking);385 }386 };387 388 render_if(input_message_type::CONTENT_ONLY, "content_only", build_content_only_message());389 render_if(input_message_type::REASONING_CONTENT, "reasoning_content", build_reasoning_content_message());390 render_if(input_message_type::TOOL_CALL_ONLY, "tool_call_only", build_tool_call_only_message());391 render_if(input_message_type::CONTENT_TOOL_CALL, "content_tool_call", build_content_tool_call_message());392 render_if(input_message_type::REASONING_TOOL_CALL, "reasoning_tool_call", build_reasoning_tool_call_message());393 render_if(input_message_type::CONTENT_FAKE_TOOL_CALL, "content_fake_tool_call",394 build_content_fake_tool_call_message());395 396 // Also render with add_generation_prompt=true to show the prompt ending397 if (message_type == input_message_type::ALL) {398 LOG_ERR("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n");399 400 json prompt_messages = json::array({ user_msg });401 render_scenario(tmpl, "generation_prompt_only", prompt_messages, tools, true, enable_thinking);402 403 // With enable_thinking toggled404 render_scenario(tmpl, "generation_prompt_thinking_disabled", prompt_messages, tools, true, false);405 }406}407 408static generation_params prepare_debug_params(const debug_options & opts, const json & tools) {409 generation_params params;410 params.messages = json::array({ build_debug_user_message() });411 params.reasoning_format = opts.enable_reasoning ? COMMON_REASONING_FORMAT_DEEPSEEK : COMMON_REASONING_FORMAT_NONE;412 params.enable_thinking = opts.enable_reasoning;413 params.add_generation_prompt = opts.generation_prompt;414 415 if (opts.with_tools) {416 params.tools = tools;417 params.tool_choice = opts.force_tool_call ? COMMON_CHAT_TOOL_CHOICE_REQUIRED : COMMON_CHAT_TOOL_CHOICE_AUTO;418 } else {419 params.tools = json();420 params.tool_choice = COMMON_CHAT_TOOL_CHOICE_NONE;421 }422 params.parallel_tool_calls = opts.parallel_tool_calls;423 return params;424}425 426static int debug_single_template(const debug_options & opts) {427 std::string template_source;428 try {429 // Check if the file is a GGUF file430 if (opts.template_path.size() >= 5 &&431 opts.template_path.compare(opts.template_path.size() - 5, 5, ".gguf") == 0) {432 template_source = read_gguf_chat_template(opts.template_path);433 } else {434 template_source = read_file(opts.template_path);435 }436 } catch (const std::exception & e) {437 LOG_ERR("Error reading template: %s\n", e.what());438 return 1;439 }440 441 LOG_ERR("Analyzing template: %s\n", opts.template_path.c_str());442 LOG_ERR("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false",443 opts.generation_prompt ? "true" : "false", opts.enable_reasoning ? "true" : "false");444 445 try {446 common_chat_template chat_template(template_source, "", "");447 448 json tools = opts.with_tools ? build_tools_definition() : json();449 450 generation_params params = prepare_debug_params(opts, tools);451 common_chat_params parser_data;452 if (std::optional<common_chat_params> spec_tmpl =453 common_chat_try_specialized_template(chat_template, template_source, params)) {454 LOG_ERR("\n");455 LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n");456 parser_data = *spec_tmpl;457 } else {458 // Render template scenarios if requested459 if (opts.input_message != input_message_type::NONE &&460 (opts.mode == output_mode::TEMPLATE || opts.mode == output_mode::BOTH)) {461 LOG_ERR("\n");462 LOG_ERR("================================================================================\n");463 LOG_ERR(" TEMPLATE RENDERING OUTPUT\n");464 LOG_ERR("================================================================================\n");465 466 render_all_scenarios(chat_template, tools, opts.generation_prompt, opts.enable_reasoning,467 opts.input_message);468 }469 470 // Output analysis if requested471 if (opts.mode == output_mode::ANALYSIS || opts.mode == output_mode::BOTH) {472 LOG_ERR("\n");473 LOG_ERR("================================================================================\n");474 LOG_ERR(" TEMPLATE ANALYSIS\n");475 LOG_ERR("================================================================================\n");476 477 struct autoparser analysis;478 analysis.analyze_template(chat_template);479 480 // Generate Parser481 parser_data = peg_generator::generate_parser(chat_template, params, analysis);482 }483 }484 485 if (!std::empty(parser_data.parser)) {486 LOG_ERR("\n=== Generated Parser ===\n");487 common_peg_arena arena;488 arena.load(parser_data.parser);489 LOG_ERR("%s\n", arena.dump(arena.root()).c_str());490 491 LOG_ERR("\n=== Generated Grammar ===\n");492 LOG_ERR("%s\n", parser_data.grammar.c_str());493 494 LOG_ERR("\n=== Generated Lazy Grammar ===\n");495 LOG_ERR("%d\n", parser_data.grammar_lazy);496 497 LOG_ERR("\n=== Generated Grammar Triggers ===\n");498 for (const common_grammar_trigger & cgt : parser_data.grammar_triggers) {499 LOG_ERR("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str());500 }501 502 LOG_ERR("\n=== Preserved Tokens ===\n");503 for (const std::string & token : parser_data.preserved_tokens) {504 LOG_ERR(" '%s'\n", token.c_str());505 }506 }507 } catch (const std::exception & e) {508 LOG_ERR("Analysis failed: %s\n", e.what());509 return 1;510 }511 512 return 0;513}514 515int main(int argc, char * argv[]) {516 if (argc > 1) {517 std::string arg = argv[1];518 if (arg == "-h" || arg == "--help") {519 common_log_set_verbosity_thold(99);520 print_usage(argv[0]);521 return 0;522 }523 524 // debug mode: if the first argument is an existing file, analyze that template instead of running the automated tests525 if (std::filesystem::is_regular_file(arg)) {526 common_log_set_verbosity_thold(99);527 528 debug_options opts;529 if (!parse_debug_options(argc, argv, opts)) {530 return 1;531 }532 533 if (opts.debug_jinja || std::getenv("LLAMA_DEBUG_JINJA") != nullptr) {534 jinja::enable_debug(true);535 }536 537 return debug_single_template(opts);538 }539 }540 541 testing t(std::cout);542 t.verbose = true;543 544 // usage: test-chat-auto-parser [filter_regex]545 546 if (argc > 1) {547 t.set_filter(argv[1]);548 }549 550 t.test("diff_split", test_calculate_diff_split);551 t.test("common_prefix", test_until_common_prefix);552 t.test("common_suffix", test_after_common_suffix);553 t.test("compare_variants", test_compare_variants);554 t.test("segments", test_marker_separation);555 t.test("seed_oss_diffs", test_seed_oss_tool_analysis);556 t.test("cohere", test_cohere_analysis);557 t.test("nemotron", test_nemotron_analysis);558 t.test("laguna", test_laguna_analysis);559 t.test("laguna-s", test_laguna_s_analysis);560 t.test("laguna-xs2", test_laguna_xs2_analysis);561 t.test("smollm3", test_smollm3_analysis);562 t.test("standard_json_tools", test_standard_json_tools_formats);563 t.test("normalize_quotes_to_json", test_normalize_quotes_to_json);564 t.test("tagged_args_embedded_quotes", test_tagged_args_with_embedded_quotes);565 t.test("bailing_v3", test_bailing_v3_tool_format);566 t.test("role_markers_all_templates", test_role_markers_all_templates);567 568 return t.summary();569}570 571static void test_marker_separation(testing & t) {572 auto single_square_marker = segmentize_markers("pre_marker[marker]post_marker");573 auto single_diag_marker = segmentize_markers("pre_marker<marker>post_marker");574 auto paired_markers = segmentize_markers("<hello>world</hello>");575 auto double_different_markers = segmentize_markers("<hello>[hello]<world>[world]");576 auto in_between = segmentize_markers("im<blue>daba<dee>da[hey]");577 578 t.test("single_square_marker", [&] (testing & t) {579 t.assert_equal("first is text", segment_type::TEXT, single_square_marker[0].type);580 t.assert_equal("second is marker", segment_type::MARKER, single_square_marker[1].type);581 t.assert_equal("last is text", segment_type::TEXT, single_square_marker[2].type);582 583 t.assert_equal("first is 'pre_marker'", "pre_marker", single_square_marker[0].value);584 t.assert_equal("second is '[marker]'", "[marker]", single_square_marker[1].value);585 t.assert_equal("last is 'post_marker'", "post_marker", single_square_marker[2].value);586 });587 588 t.test("single_diagonal_marker", [&] (testing & t) {589 t.assert_equal("first is text", segment_type::TEXT, single_diag_marker[0].type);590 t.assert_equal("second is marker", segment_type::MARKER, single_diag_marker[1].type);591 t.assert_equal("last is text", segment_type::TEXT, single_diag_marker[2].type);592 593 t.assert_equal("first is 'pre_marker'", "pre_marker", single_diag_marker[0].value);594 t.assert_equal("second is '<marker>'", "<marker>", single_diag_marker[1].value);595 t.assert_equal("last is 'post_marker'", "post_marker", single_diag_marker[2].value);596 });597 598 t.test("paired_markers", [&] (testing & t) {599 t.assert_equal("first is marker", segment_type::MARKER, paired_markers[0].type);600 t.assert_equal("second is text", segment_type::TEXT, paired_markers[1].type);601 t.assert_equal("third is marker", segment_type::MARKER, paired_markers[2].type);602 603 t.assert_equal("first is '<hello>'", "<hello>", paired_markers[0].value);604 t.assert_equal("second is 'world'", "world", paired_markers[1].value);605 t.assert_equal("third is '</hello>'", "</hello>", paired_markers[2].value);606 });607 608 t.test("double_different_markers", [&] (testing & t) {609 t.assert_equal("first is marker", segment_type::MARKER, double_different_markers[0].type);610 t.assert_equal("second is marker", segment_type::MARKER, double_different_markers[1].type);611 t.assert_equal("third is marker", segment_type::MARKER, double_different_markers[2].type);612 t.assert_equal("fourth is marker", segment_type::MARKER, double_different_markers[3].type);613 614 t.assert_equal("first is '<hello>'", "<hello>", double_different_markers[0].value);615 t.assert_equal("second is '[hello]'", "[hello]", double_different_markers[1].value);616 t.assert_equal("third is '<world>'", "<world>", double_different_markers[2].value);617 t.assert_equal("fourth is '[world]'", "[world]", double_different_markers[3].value);618 });619 620 t.test("in_between", [&] (testing & t) {621 t.assert_equal("first is text", segment_type::TEXT, in_between[0].type);622 t.assert_equal("second is marker", segment_type::MARKER, in_between[1].type);623 t.assert_equal("third is text", segment_type::TEXT, in_between[2].type);624 t.assert_equal("fourth is marker", segment_type::MARKER, in_between[3].type);625 t.assert_equal("fifth is text", segment_type::TEXT, in_between[4].type);626 t.assert_equal("sixth is marker", segment_type::MARKER, in_between[5].type);627 628 t.assert_equal("first is 'im'", "im", in_between[0].value);629 t.assert_equal("second is '<blue>'", "<blue>", in_between[1].value);630 t.assert_equal("third is 'daba'", "daba", in_between[2].value);631 t.assert_equal("fourth is '<dee>'", "<dee>", in_between[3].value);632 t.assert_equal("fifth is 'da'", "da", in_between[4].value);633 t.assert_equal("sixth is '[hey]'", "[hey]", in_between[5].value);634 });635}636 637static void test_calculate_diff_split(testing & t) {638 t.test("calculate_diff_split basic", test_calculate_diff_split_basic);639 t.test("calculate_diff_split identical", test_calculate_diff_split_identical);640 t.test("calculate_diff_split common prefix", test_calculate_diff_split_common_prefix);641 t.test("calculate_diff_split common suffix", test_calculate_diff_split_common_suffix);642 t.test("calculate_diff_split common both", test_calculate_diff_split_common_both);643 t.test("calculate_diff_split empty cases", test_calculate_diff_split_empty_cases);644 t.test("calculate_diff_split no common", test_calculate_diff_split_no_common);645 t.test("calculate_diff_split single char", test_calculate_diff_split_single_char);646 t.test("calculate_diff_split overlaps", test_calculate_diff_split_overlaps);647 t.test("calculate_diff_split tag boundaries", test_calculate_diff_split_tag_boundaries);648 t.test("calculate_diff_split generation prompt", test_calculate_diff_split_generation_prompt);649}650 651static void test_calculate_diff_split_basic(testing & t) {652 diff_split result = calculate_diff_split("hello world", "hello test");653 t.assert_equal("prefix should be 'hello '", "hello ", result.prefix);654 t.assert_equal("left should be 'world'", "world", result.left);655 t.assert_equal("right should be 'test'", "test", result.right);656 t.assert_equal("suffix should be empty", "", result.suffix);657 658 result = calculate_diff_split("abc", "xyz");659 t.assert_equal("prefix should be empty", "", result.prefix);660 t.assert_equal("left should be 'abc'", "abc", result.left);661 t.assert_equal("right should be 'xyz'", "xyz", result.right);662 t.assert_equal("suffix should be empty", "", result.suffix);663 664 result = calculate_diff_split("prefixA suffix", "prefixB suffix");665 t.assert_equal("prefix should be 'prefix'", "prefix", result.prefix);666 t.assert_equal("left should be 'A'", "A", result.left);667 t.assert_equal("right should be 'B'", "B", result.right);668 t.assert_equal("suffix should be ' suffix'", " suffix", result.suffix);669}670 671static void test_calculate_diff_split_identical(testing & t) {672 diff_split result = calculate_diff_split("hello", "hello");673 t.assert_equal("prefix should be 'hello'", "hello", result.prefix);674 t.assert_equal("left should be empty", "", result.left);675 t.assert_equal("right should be empty", "", result.right);676 t.assert_equal("suffix should be empty", "", result.suffix);677 678 result = calculate_diff_split("", "");679 t.assert_equal("prefix should be empty", "", result.prefix);680 t.assert_equal("left should be empty", "", result.left);681 t.assert_equal("right should be empty", "", result.right);682 t.assert_equal("suffix should be empty", "", result.suffix);683 684 result = calculate_diff_split("a", "a");685 t.assert_equal("prefix should be 'a'", "a", result.prefix);686 t.assert_equal("left should be empty", "", result.left);687 t.assert_equal("right should be empty", "", result.right);688 t.assert_equal("suffix should be empty", "", result.suffix);689 690 result = calculate_diff_split("<row><row><row><your><boat><gently>", "<row><row><row><your><boat><gently>");691 t.assert_equal("prefix should be '<row><row><row><your><boat><gently>'", "<row><row><row><your><boat><gently>", result.prefix);692 t.assert_equal("left should be empty", "", result.left);693 t.assert_equal("right should be empty", "", result.right);694 t.assert_equal("suffix should be empty", "", result.suffix);695}696 697static void test_calculate_diff_split_common_prefix(testing & t) {698 diff_split result = calculate_diff_split("abcdef", "abcxyz");699 t.assert_equal("prefix should be 'abc'", "abc", result.prefix);700 t.assert_equal("left should be 'def'", "def", result.left);701 t.assert_equal("right should be 'xyz'", "xyz", result.right);702 t.assert_equal("suffix should be empty", "", result.suffix);703 704 result = calculate_diff_split("same", "sameagain");705 t.assert_equal("prefix should be 'same'", "same", result.prefix);706 t.assert_equal("left should be empty", "", result.left);707 t.assert_equal("right should be 'again'", "again", result.right);708 t.assert_equal("suffix should be empty", "", result.suffix);709 710 result = calculate_diff_split("test", "testing");711 t.assert_equal("prefix should be 'test'", "test", result.prefix);712 t.assert_equal("left should be empty", "", result.left);713 t.assert_equal("right should be 'ing'", "ing", result.right);714 t.assert_equal("suffix should be empty", "", result.suffix);715}716 717static void test_calculate_diff_split_common_suffix(testing & t) {718 diff_split result = calculate_diff_split("123end", "456end");719 t.assert_equal("prefix should be empty", "", result.prefix);720 t.assert_equal("left should be '123'", "123", result.left);721 t.assert_equal("right should be '456'", "456", result.right);722 t.assert_equal("suffix should be 'end'", "end", result.suffix);723 724 result = calculate_diff_split("start", "end");725 t.assert_equal("prefix should be empty", "", result.prefix);726 t.assert_equal("left should be 'start'", "start", result.left);727 t.assert_equal("right should be 'end'", "end", result.right);728 t.assert_equal("suffix should be empty", "", result.suffix);729 730 result = calculate_diff_split("abcsuffix", "xyzsuffix");731 t.assert_equal("prefix should be empty", "", result.prefix);732 t.assert_equal("left should be 'abc'", "abc", result.left);733 t.assert_equal("right should be 'xyz'", "xyz", result.right);734 t.assert_equal("suffix should be 'suffix'", "suffix", result.suffix);735}736 737static void test_calculate_diff_split_common_both(testing & t) {738 diff_split result = calculate_diff_split("helloXworld", "helloYworld");739 t.assert_equal("prefix should be 'hello'", "hello", result.prefix);740 t.assert_equal("left should be 'X'", "X", result.left);741 t.assert_equal("right should be 'Y'", "Y", result.right);742 t.assert_equal("suffix should be 'world'", "world", result.suffix);743 744 result = calculate_diff_split("ABCmiddleXYZ", "ABCdifferentXYZ");745 t.assert_equal("prefix should be 'ABC'", "ABC", result.prefix);746 t.assert_equal("left should be 'middle'", "middle", result.left);747 t.assert_equal("right should be 'different'", "different", result.right);748 t.assert_equal("suffix should be 'XYZ'", "XYZ", result.suffix);749 750 result = calculate_diff_split("startAend", "startBend");751 t.assert_equal("prefix should be 'start'", "start", result.prefix);752 t.assert_equal("left should be 'A'", "A", result.left);753 t.assert_equal("right should be 'B'", "B", result.right);754 t.assert_equal("suffix should be 'end'", "end", result.suffix);755 756 // Edge case: common prefix and suffix overlap757 result = calculate_diff_split("aa", "ab");758 t.assert_equal("prefix should be 'a'", "a", result.prefix);759 t.assert_equal("left should be 'a'", "a", result.left);760 t.assert_equal("right should be 'b'", "b", result.right);761 t.assert_equal("suffix should be empty", "", result.suffix);762}763 764static void test_calculate_diff_split_empty_cases(testing & t) {765 // Empty left, non-empty right766 diff_split result = calculate_diff_split("", "hello");767 t.assert_equal("prefix should be empty", "", result.prefix);768 t.assert_equal("left should be empty", "", result.left);769 t.assert_equal("right should be 'hello'", "hello", result.right);770 t.assert_equal("suffix should be empty", "", result.suffix);771 772 // Non-empty left, empty right773 result = calculate_diff_split("hello", "");774 t.assert_equal("prefix should be empty", "", result.prefix);775 t.assert_equal("left should be 'hello'", "hello", result.left);776 t.assert_equal("right should be empty", "", result.right);777 t.assert_equal("suffix should be empty", "", result.suffix);778 779 // Both empty780 result = calculate_diff_split("", "");781 t.assert_equal("prefix should be empty", "", result.prefix);782 t.assert_equal("left should be empty", "", result.left);783 t.assert_equal("right should be empty", "", result.right);784 t.assert_equal("suffix should be empty", "", result.suffix);785 786 // Left single char, empty right787 result = calculate_diff_split("a", "");788 t.assert_equal("prefix should be empty", "", result.prefix);789 t.assert_equal("left should be 'a'", "a", result.left);790 t.assert_equal("right should be empty", "", result.right);791 t.assert_equal("suffix should be empty", "", result.suffix);792 793 // Empty left, right single char794 result = calculate_diff_split("", "a");795 t.assert_equal("prefix should be empty", "", result.prefix);796 t.assert_equal("left should be empty", "", result.left);797 t.assert_equal("right should be 'a'", "a", result.right);798 t.assert_equal("suffix should be empty", "", result.suffix);799}800 801static void test_calculate_diff_split_no_common(testing & t) {802 diff_split result = calculate_diff_split("abc", "xyz");803 t.assert_equal("prefix should be empty", "", result.prefix);804 t.assert_equal("left should be 'abc'", "abc", result.left);805 t.assert_equal("right should be 'xyz'", "xyz", result.right);806 t.assert_equal("suffix should be empty", "", result.suffix);807 808 result = calculate_diff_split("left", "right");809 // The algorithm finds "t" as a common suffix since both strings end with 't'810 // This is the algorithm's actual behavior - it finds maximal common suffix811 t.assert_equal("prefix should be empty", "", result.prefix);812 t.assert_equal("left should be 'lef'", "lef", result.left);813 t.assert_equal("right should be 'righ'", "righ", result.right);814 t.assert_equal("suffix should be 't'", "t", result.suffix);815 816 result = calculate_diff_split("123", "456");817 t.assert_equal("prefix should be empty", "", result.prefix);818 t.assert_equal("left should be '123'", "123", result.left);819 t.assert_equal("right should be '456'", "456", result.right);820 t.assert_equal("suffix should be empty", "", result.suffix);821}822 823static void test_calculate_diff_split_single_char(testing & t) {824 diff_split result = calculate_diff_split("a", "b");825 t.assert_equal("prefix should be empty", "", result.prefix);826 t.assert_equal("left should be 'a'", "a", result.left);827 t.assert_equal("right should be 'b'", "b", result.right);828 t.assert_equal("suffix should be empty", "", result.suffix);829 830 result = calculate_diff_split("a", "a");831 t.assert_equal("prefix should be 'a'", "a", result.prefix);832 t.assert_equal("left should be empty", "", result.left);833 t.assert_equal("right should be empty", "", result.right);834 t.assert_equal("suffix should be empty", "", result.suffix);835 836 result = calculate_diff_split("a", "ab");837 t.assert_equal("prefix should be 'a'", "a", result.prefix);838 t.assert_equal("left should be empty", "", result.left);839 t.assert_equal("right should be 'b'", "b", result.right);840 t.assert_equal("suffix should be empty", "", result.suffix);841 842 result = calculate_diff_split("ab", "a");843 t.assert_equal("prefix should be 'a'", "a", result.prefix);844 t.assert_equal("left should be 'b'", "b", result.left);845 t.assert_equal("right should be empty", "", result.right);846 t.assert_equal("suffix should be empty", "", result.suffix);847}848 849static void test_calculate_diff_split_overlaps(testing & t) {850 // One string is substring of another851 diff_split result = calculate_diff_split("test", "testing");852 t.assert_equal("prefix should be 'test'", "test", result.prefix);853 t.assert_equal("left should be empty", "", result.left);854 t.assert_equal("right should be 'ing'", "ing", result.right);855 t.assert_equal("suffix should be empty", "", result.suffix);856 857 result = calculate_diff_split("testing", "test");858 t.assert_equal("prefix should be 'test'", "test", result.prefix);859 t.assert_equal("left should be 'ing'", "ing", result.left);860 t.assert_equal("right should be empty", "", result.right);861 t.assert_equal("suffix should be empty", "", result.suffix);862 863 // Similar strings with one extra char at start864 result = calculate_diff_split("Xtest", "Ytest");865 // The algorithm finds "test" as a common suffix since both strings end with "test"866 // This is the algorithm's actual behavior - it finds maximal common suffix867 t.assert_equal("prefix should be empty", "", result.prefix);868 t.assert_equal("left should be 'X'", "X", result.left);869 t.assert_equal("right should be 'Y'", "Y", result.right);870 t.assert_equal("suffix should be 'test'", "test", result.suffix);871 872 // Similar strings with one extra char at end873 result = calculate_diff_split("testX", "testY");874 t.assert_equal("prefix should be 'test'", "test", result.prefix);875 t.assert_equal("left should be 'X'", "X", result.left);876 t.assert_equal("right should be 'Y'", "Y", result.right);877 t.assert_equal("suffix should be empty", "", result.suffix);878 879 // Strings that are reverses880 result = calculate_diff_split("abc", "cba");881 t.assert_equal("prefix should be empty", "", result.prefix);882 t.assert_equal("left should be 'abc'", "abc", result.left);883 t.assert_equal("right should be 'cba'", "cba", result.right);884 t.assert_equal("suffix should be empty", "", result.suffix);885}886 887static void test_calculate_diff_split_tag_boundaries(testing & t) {888 // Test with unclosed XML tags889 diff_split result = calculate_diff_split("test<tag", "test>content");890 // The fix_tag_boundaries should move incomplete tags appropriately891 t.assert_true("prefix should start with 'test'", result.prefix.find("test") == 0);892 t.assert_true("should handle tag boundaries", result.left != "" || result.right != "" || result.suffix != "");893 894 // Test with unclosed brackets895 result = calculate_diff_split("test[", "test]value");896 t.assert_true("should handle bracket boundaries", result.left != "" || result.right != "" || result.suffix != "");897 898 // Test with partial tags on both sides899 result = calculate_diff_split("prefix<tag>", "prefix</tag>suffix");900 // fix_tag_boundaries moves the incomplete '<' from prefix to left/right901 t.assert_equal("prefix should be 'prefix'", "prefix", result.prefix);902 t.assert_equal("left should be '<tag>'", "<tag>", result.left);903 t.assert_equal("right should be '</tag>suffix'", "</tag>suffix", result.right);904 t.assert_equal("suffix should be empty", "", result.suffix);905 906 // Test with complex nested tags907 result = calculate_diff_split("prefix<div>content</div>", "prefix<div>different</div>");908 // Algorithm finds "ent</div>" as a common suffix because both strings end with it909 // This is the actual algorithm behavior, though not semantically ideal910 t.assert_equal("prefix should be 'prefix<div>'", "prefix<div>", result.prefix);911 t.assert_equal("left should be 'cont'", "cont", result.left);912 t.assert_equal("right should be 'differ'", "differ", result.right);913 t.assert_equal("suffix should be 'ent</div>'", "ent</div>", result.suffix);914 915 // Test with unclosed angle bracket916 result = calculate_diff_split("Hello <world>", "Hello test");917 t.assert_equal("prefix should be 'Hello '", "Hello ", result.prefix);918 t.assert_true("left should contain '<world>'", result.left.find("<world>") != std::string::npos);919 t.assert_equal("right should be 'test'", "test", result.right);920 t.assert_equal("suffix should be empty", "", result.suffix);921 922 // Test with unclosed square bracket923 result = calculate_diff_split("test [array]", "test other");924 t.assert_equal("prefix should be 'test '", "test ", result.prefix);925 t.assert_true("left should contain '[array]'", result.left.find("[array]") != std::string::npos);926 t.assert_equal("right should be 'other'", "other", result.right);927 t.assert_equal("suffix should be empty", "", result.suffix);928 929 // Test empty prefix and suffix with tags930 result = calculate_diff_split("<tag>left</tag>", "<tag>righ</tag>");931 t.assert_equal("prefix should be '<tag>'", "<tag>", result.prefix);932 t.assert_equal("left should be 'left'", "left", result.left);933 t.assert_equal("right should be 'righ'", "righ", result.right);934 t.assert_equal("suffix should be '</tag>'", "</tag>", result.suffix);935 936 {937 // real case from template tests, simplified938 std::string left = "PREFIX</think>Sure";939 std::string right = "PREFIX<think>Lemme think</think>Sure";940 result = calculate_diff_split(left, right);941 t.assert_equal("prefix should be PREFIX", "PREFIX", result.prefix);942 t.assert_equal("suffix should be </think>Sure", "</think>Sure", result.suffix);943 t.assert_equal("left should be empty", "", result.left);944 t.assert_equal("right should be <think>Lemme think", "<think>Lemme think", result.right);945 }946 947 {948 // Real case: special tokens with |> boundary issue949 // The suffix starts with |> which should be moved to complete <|END_RESPONSE and <|END_ACTION950 std::string prefix = "SOME_PREFIX";951 std::string suffix = "|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>";952 std::string left_diff = "<|START_RESPONSE|>Let me help you.<|END_RESPONSE";953 std::string right_diff =954 "<|START_THINKING|><|END_THINKING|><|START_ACTION|>[\n"955 " {\"tool_call_id\": \"0\", \"tool_name\": \"test_function_name\", "956 "\"parameters\": {\"param1\": \"value1\", \"param2\": \"value2\"}}\n"957 "]<|END_ACTION";958 959 std::string left = prefix + left_diff + suffix;960 std::string right = prefix + right_diff + suffix;961 result = calculate_diff_split(left, right);962 963 t.assert_equal("special token prefix", prefix, result.prefix);964 // The |> should be moved from suffix to complete the tokens965 t.assert_equal("special token left", "<|START_RESPONSE|>Let me help you.<|END_RESPONSE|>", result.left);966 t.assert_true("special token right ends with |>", result.right.find("<|END_ACTION|>") != std::string::npos);967 t.assert_equal("special token suffix", "<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",968 result.suffix);969 }970}971 972static void test_calculate_diff_split_generation_prompt(testing & t) {973 // ChatML thinking template: left is a prefix of right, generation_prompt is the appended part.974 // The trailing \n in left matches the trailing \n in the generation_prompt, causing975 // the suffix matcher to steal it and rotate the diff result.976 {977 // Simplified reproduction: left ends with \n, right = left + "<|im_start|>assistant\n<think>\n"978 std::string left = "<|im_start|>user\nHello<|im_end|>\n";979 std::string right = left + "<|im_start|>assistant\n<think>\n";980 diff_split result = calculate_diff_split(left, right);981 t.assert_equal("chatml prefix", left, result.prefix);982 t.assert_equal("chatml left", "", result.left);983 t.assert_equal("chatml right should be generation prompt",984 "<|im_start|>assistant\n<think>\n", result.right);985 t.assert_equal("chatml suffix", "", result.suffix);986 }987 988 {989 // More realistic: longer conversation ending with tool_response990 std::string common =991 "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"992 "<|im_start|>user\nSearch for files<|im_end|>\n"993 "<|im_start|>assistant\n<think>\nLet me search.\n</think>\n\n"994 "<tool_call>\n<function=search>\n</function>\n</tool_call><|im_end|>\n"995 "<|im_start|>user\n<tool_response>\nNo files found\n</tool_response><|im_end|>\n";996 std::string left = common;997 std::string right = common + "<|im_start|>assistant\n<think>\n";998 diff_split result = calculate_diff_split(left, right);999 t.assert_equal("tool_response left", "", result.left);1000 t.assert_equal("tool_response right should be generation prompt",1001 "<|im_start|>assistant\n<think>\n", result.right);1002 }1003}1004 1005static void test_until_common_prefix(testing & t) {1006 t.test("until_common_prefix basic", test_until_common_prefix_basic);1007}1008 1009static void test_until_common_prefix_basic(testing & t) {1010 // Test case from the user request1011 std::string result = until_common_prefix("<function name=foo><arg name=bar>", "<arg name=bar>", "<arg name=baz>");1012 t.assert_equal("untilCommonPrefix should return '<function name=foo>'", "<function name=foo>", result);1013 1014 // Additional test cases to ensure robustness1015 // Test with different common prefix lengths1016 result = until_common_prefix("prefix<test>suffix", "<test>different", "<test>other");1017 t.assert_equal("should return 'prefix'", "prefix", result);1018 1019 // Test when common prefix is at the start1020 result = until_common_prefix("<common>rest", "<common>left", "<common>right");1021 t.assert_equal("should return empty string when common prefix at start", "", result);1022 1023 // Test when there's no common prefix1024 result = until_common_prefix("something", "left", "right");1025 t.assert_equal("should return empty string when no common prefix", "", result);1026 1027 // Test with empty strings1028 result = until_common_prefix("test", "", "right");1029 t.assert_equal("should return empty string when left is empty", "", result);1030 1031 // Test with longer common prefix1032 result = until_common_prefix("abcXYZ<shared_prefix>rest", "<shared_prefix>left", "<shared_prefix>right");1033 t.assert_equal("should return 'abcXYZ'", "abcXYZ", result);1034}1035 1036static void test_after_common_suffix(testing & t) {1037 t.test("after_common_suffix basic", test_after_common_suffix_basic);1038}1039 1040static void test_after_common_suffix_basic(testing & t) {1041 // Test case from the user request1042 std::string result = after_common_suffix("<function name=foo><arg name=bar>100</arg></function>",1043 "<arg name=bar>100</arg>",1044 "<arg name=baz>535</arg>");1045 t.assert_equal("afterCommonSuffix should return '</function>'", "</function>", result);1046 1047 // Test when common suffix is at the end1048 result = after_common_suffix("rest<common>", "left<common>", "right<common>");1049 t.assert_equal("should return empty string when common suffix at end", "", result);1050 1051 // Test with empty strings1052 result = after_common_suffix("test", "left", "");1053 t.assert_equal("should return empty string when right is empty", "", result);1054 1055 // Test case with XML-like structure similar to the main example1056 result = after_common_suffix("<outer><inner>value</inner></outer>",1057 "<inner>value</inner>",1058 "<inner>different</inner>");1059 t.assert_equal("should return '</outer>'", "</outer>", result);1060 1061 // Test with longer common suffix appearing at the end of full1062 result = after_common_suffix("prefix<shared>rest</shared>", "prefix<shared>left</shared>", "prefix<shared>right</shared>");1063 t.assert_equal("should return '' when common suffix is at end of full", "", result);1064 1065 // Test with common suffix appearing in middle but not at end1066 result = after_common_suffix("<tag>content</tag><extra>", "<tag>value</tag>", "<tag>other</tag>");1067 t.assert_equal("should return '<extra>' when common suffix appears before end", "<extra>", result);1068 1069 // Test with multi-character common suffix at the very end of full1070 result = after_common_suffix("start<middle>end</middle>", "prefix<middle>left</middle>", "prefix<middle>right</middle>");1071 t.assert_equal("should return '' when common suffix </middle> is at end of full", "", result);1072}1073 1074static void test_compare_variants(testing & t) {1075 t.test("compare_variants basic", test_compare_variants_basic);1076 t.test("compare_variants messages modifier", test_compare_variants_messages_modifier);1077 t.test("compare_variants tools modifier", test_compare_variants_tools_modifier);1078 t.test("compare_variants both modifiers", test_compare_variants_both_modifiers);1079 t.test("compare_variants template failure", test_compare_variants_template_failure);1080 t.test("compare_variants identity", test_compare_variants_identity);1081}1082 1083static void test_compare_variants_basic(testing & t) {1084 // Create a simple template that just echoes messages1085 common_chat_template tmpl("{{ messages[0]['content'] }}", "", "");1086 1087 template_params params;1088 params.messages = json::array({1089 json {{"role", "user"}, {"content", "Hello"}}1090 });1091 1092 auto modifier = [](template_params & p) {1093 p.messages[0]["content"] = "World";1094 };1095 1096 auto result = ::compare_variants(tmpl, params, modifier);1097 1098 if (!t.assert_true("result should have value", result.has_value())) {1099 return;1100 }1101 // The template might not output anything if messages is empty or format is different1102 // Check that we get a valid result1103 t.assert_true("prefix or left should have content", !result->diff.prefix.empty() || !result->diff.left.empty());1104}1105 1106static void test_compare_variants_messages_modifier(testing & t) {1107 // Test with messages modifier only1108 common_chat_template tmpl("{% for message in messages %}{{ message['role'] }}:{{ message['content'] }}{% endfor %}", "", "");1109 1110 template_params params;1111 params.messages = json::array({1112 json {{"role", "user"}, {"content", "A"}}1113 });1114 1115 auto modifier = [](template_params & p) {1116 p.messages[0]["content"] = "B";1117 };1118 1119 std::optional<compare_variants_result> result = ::compare_variants(tmpl, params, modifier);1120 1121 if (!t.assert_true("result should have value", result.has_value())) {1122 return;1123 }1124 t.assert_equal("left should be 'A'", "A", result->diff.left);1125 t.assert_equal("right should be 'B'", "B", result->diff.right);1126}1127 1128static void test_compare_variants_tools_modifier(testing & t) {1129 // Test with tools modifier only1130 common_chat_template tmpl(1131 "{% for tool in tools %}{{ tool['name'] }}{% endfor %}", "", "");1132 1133 template_params params;1134 params.tools = json::array({1135 json {{"name", "foo"}}1136 });1137 1138 auto modifier = [](template_params & p) {1139 p.tools[0]["name"] = "bar";1140 };1141 1142 auto result = ::compare_variants(tmpl, params, modifier);1143 1144 if (!t.assert_true("result should have value", result.has_value())) {1145 return;1146 }1147 t.assert_equal("left should be 'foo'", "foo", result->diff.left);1148 t.assert_equal("right should be 'bar'", "bar", result->diff.right);1149}1150 1151static void test_compare_variants_both_modifiers(testing & t) {1152 // Test with both messages and tools modifiers using the for loop approach1153 common_chat_template tmpl(1154 "{% for message in messages %}{{ message['role'] }}:{{ message['content'] }}{% endfor %}", "", "");1155 1156 template_params params;1157 params.messages = json::array({1158 json {{"role", "user"}, {"content", "A"}}1159 });1160 1161 auto modifier = [](template_params & p) {1162 p.messages[0]["content"] = "B";1163 p.messages[0]["role"] = "newuser";1164 };1165 1166 auto result = ::compare_variants(tmpl, params, modifier);1167 1168 if (!t.assert_true("result should have value", result.has_value())) {1169 return;1170 }1171 t.assert_equal("left should be 'user:A'", "user:A", result->diff.left);1172 t.assert_equal("right should be 'newuser:B'", "newuser:B", result->diff.right);1173}1174 1175static void test_compare_variants_template_failure(testing & t) {1176 // Test with template that causes failure during application (not construction)1177 // We use a valid template syntax but one that will fail during application1178 common_chat_template tmpl("{{ messages.cahoot()[0]['nonexistent_field'] }}", "", "");1179 1180 template_params params;1181 params.messages = json::array({1182 json {{"role", "user"}, {"content", "Hello"}}1183 });1184 1185 auto modifier = [](template_params & p) {1186 p.messages[0]["content"] = "World";1187 };1188 1189 auto result = ::compare_variants(tmpl, params, modifier);1190 1191 t.assert_true("result should be nullopt on template failure", !result.has_value());1192}1193 1194static void test_compare_variants_identity(testing & t) {1195 // Test with identity modifier (no change)1196 common_chat_template tmpl("{{ messages[0]['content'] }}", "", "");1197 1198 template_params params;1199 params.messages = json::array({1200 json {{"role", "user"}, {"content", "Hello"}}