Felipe97/llama-cpp-compiled
01.1k
1#include "chat-auto-parser.h"2#include "chat-auto-parser-helpers.h"3#include "chat.h"4#include "log.h"5#include "jinja/caps.h"6#include "jinja/runtime.h"7 8#include <fstream>9#include <sstream>10#include <string>11#include <vector>12#include <algorithm>13 14#include "json.h"15 16using json = common_json;17 18// ANSI color codes - using 256-color palette for brighter colors (all bold)19#define ANSI_RESET "\033[0m"20#define ANSI_PURPLE "\033[1m\x1b[38;5;126m" // Bold bright purple for main headers21#define ANSI_CYAN "\033[1m\x1b[38;5;81m" // Bold bright cyan for section headers22#define ANSI_BLUE "\033[1m\x1b[38;5;12m" // Bold bright blue for labels23#define ANSI_ORANGE "\033[1m\x1b[38;5;209m" // Bold orange for right differences24#define ANSI_GREEN "\033[1m\x1b[38;5;83m" // Bold bright green for left differences25#define ANSI_GRAY "\033[1m\x1b[38;5;240m" // Bold gray (used for "no variables" message)26#define ANSI_BOLD "\033[1m" // Standalone bold27#define ANSI_PREFIX "\033[1m\x1b[38;5;176m" // Bold color for common prefix28#define ANSI_SUFFIX "\033[1m\x1b[38;5;61m" // Bold color for common suffix29 30// All template paths extracted from tests/test-chat.cpp31static const std::vector<std::string> ALL_TEMPLATE_PATHS = {32 "models/templates/Apertus-8B-Instruct.jinja",33 "models/templates/Apriel-1.6-15b-Thinker-fixed.jinja",34 "models/templates/ByteDance-Seed-OSS.jinja",35 "models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja",36 "models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja",37 "models/templates/GLM-4.6.jinja",38 "models/templates/GLM-4.7-Flash.jinja",39 "models/templates/Kimi-K2-Instruct.jinja",40 "models/templates/Kimi-K2-Thinking.jinja",41 "models/templates/MiMo-VL.jinja",42 "models/templates/MiniMax-M2.jinja",43 "models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja",44 "models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja",45 "models/templates/NVIDIA-Nemotron-Nano-v2.jinja",46 "models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja",47 "models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja",48 "models/templates/Qwen-QwQ-32B.jinja",49 "models/templates/Qwen-Qwen2.5-7B-Instruct.jinja",50 "models/templates/Qwen3-Coder.jinja",51 "models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja",52 "models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja",53 "models/templates/deepseek-ai-DeepSeek-V3.1.jinja",54 "models/templates/fireworks-ai-llama-3-firefunction-v2.jinja",55 "models/templates/google-gemma-2-2b-it.jinja",56 "models/templates/ibm-granite-granite-3.3-2B-Instruct.jinja",57 "models/templates/llama-cpp-deepseek-r1.jinja",58 "models/templates/meetkai-functionary-medium-v3.1.jinja",59 "models/templates/meetkai-functionary-medium-v3.2.jinja",60 "models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja",61 "models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja",62 "models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja",63 "models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja",64 "models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja",65 "models/templates/moonshotai-Kimi-K2.jinja",66 "models/templates/openai-gpt-oss-120b.jinja",67 "models/templates/unsloth-Apriel-1.5.jinja",68 "models/templates/unsloth-mistral-Devstral-Small-2507.jinja",69};70 71struct analysis_options {72 std::vector<std::string> template_paths;73 bool analyze_all = false;74};75 76static std::string read_file(const std::string & path) {77 std::ifstream fin(path, std::ios::binary);78 if (!fin.is_open()) {79 throw std::runtime_error("Could not open file: " + path);80 }81 std::ostringstream buf;82 buf << fin.rdbuf();83 return buf.str();84}85 86static void print_usage(const char * program_name) {87 LOG_ERR("Debug the auto-parser's differential analysis: render a template with/without tools, reasoning, etc. and show the diffs.\n");88 LOG_ERR("\nUsage: %s [options]\n", program_name);89 LOG_ERR("\nOptions:\n");90 LOG_ERR(" --template <name> Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n");91 LOG_ERR(" --template-file <path> Analyze custom template file\n");92 LOG_ERR(" --all Analyze all templates from test suite (default when no arguments are given)\n");93 LOG_ERR("\nExamples:\n");94 LOG_ERR(" %s --all\n", program_name);95 LOG_ERR(" %s --template deepseek\n", program_name);96 LOG_ERR(" %s --template-file my-template.jinja\n", program_name);97}98 99static bool parse_options(int argc, char ** argv, analysis_options & opts) {100 if (argc < 2) {101 // default mode: analyze all templates from the test suite102 opts.analyze_all = true;103 }104 105 for (int i = 1; i < argc; ++i) {106 std::string arg = argv[i];107 108 if (arg == "-h" || arg == "--help") {109 print_usage(argv[0]);110 return false;111 } else if (arg == "--all") {112 opts.analyze_all = true;113 } else if (arg == "--template") {114 if (i + 1 >= argc) {115 LOG_ERR("--template requires an argument\n");116 return false;117 }118 std::string pattern = argv[++i];119 std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);120 121 // Find matching templates122 bool found = false;123 for (const auto & path : ALL_TEMPLATE_PATHS) {124 std::string path_lower = path;125 std::transform(path_lower.begin(), path_lower.end(), path_lower.begin(), ::tolower);126 if (path_lower.find(pattern) != std::string::npos) {127 opts.template_paths.push_back(path);128 found = true;129 }130 }131 132 if (!found) {133 LOG_ERR("No templates found matching: %s\n", pattern.c_str());134 return false;135 }136 } else if (arg == "--template-file") {137 if (i + 1 >= argc) {138 LOG_ERR("--template-file requires an argument\n");139 return false;140 }141 opts.template_paths.push_back(argv[++i]);142 } else {143 LOG_ERR("Unknown option: %s\n", arg.c_str());144 print_usage(argv[0]);145 return false;146 }147 }148 149 if (opts.analyze_all) {150 opts.template_paths = ALL_TEMPLATE_PATHS;151 }152 153 if (opts.template_paths.empty()) {154 LOG_ERR("No templates specified\n");155 print_usage(argv[0]);156 return false;157 }158 159 return true;160}161 162static json build_tools_definition() {163 json parameters_schema = json::object();164 parameters_schema["type"] = "object";165 parameters_schema["properties"] = json::object();166 parameters_schema["properties"]["param1"] = json::object({167 { "type", "string" },168 { "description", "First parameter" }169 });170 parameters_schema["properties"]["param2"] = json::object({171 { "type", "string" },172 { "description", "Second parameter" }173 });174 parameters_schema["required"] = json::array({ "param1", "param2" });175 176 return json::array({177 json{ { "type", "function" },178 { "function", json{ { "name", "test_function_name" },179 { "description", "A test function for debugging" },180 { "parameters", parameters_schema } } } }181 });182}183 184// Helper to create a tool call with arguments as JSON object185static json build_tool_call(const std::string & name, const json & args_object, const std::string & id = "call_001") {186 return json{187 {"id", id},188 {"type", "function"},189 {"function", json{190 {"name", name},191 {"arguments", args_object} // Pass as JSON object, not serialized string192 }}193 };194}195 196// Helper functions to create repeating message definitions197static json make_user_msg() {198 return json{199 {"role", "user"},200 {"content", "Hello, please help me."}201 };202}203 204static json make_user_msg2() {205 return json{206 {"role", "user"},207 {"content", "Thank you."}208 };209}210 211static json make_user_msg2_continue() {212 return json{213 {"role", "user"},214 {"content", "Continue."}215 };216}217 218static json make_assistant_no_tool() {219 return json{220 {"role", "assistant"},221 {"content", "Let me help you."}222 };223}224 225static json make_assistant_one_tool() {226 return json{227 {"role", "assistant"},228 {"content", nullptr},229 {"tool_calls", json::array({230 build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))231 })}232 };233}234 235static json make_assistant_two_tools() {236 return json{237 {"role", "assistant"},238 {"content", nullptr},239 {"tool_calls", json::array({240 build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})),241 build_tool_call("test_function_name", json::object({{"param1", "value3"}, {"param2", "value4"}}), "call_002")242 })}243 };244}245 246static json make_assistant_no_reasoning() {247 return json{248 {"role", "assistant"},249 {"content", "I can help you with that."}250 };251}252 253static json make_assistant_with_reasoning() {254 return json{255 {"role", "assistant"},256 {"content", "I can help you with that."},257 {"reasoning_content", "The user is asking for help. I should respond positively."}258 };259}260 261static json make_assistant_one_tool_with_reasoning() {262 return json{263 {"role", "assistant"},264 {"content", nullptr},265 {"tool_calls", json::array({266 build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))267 })},268 {"reasoning_content", "I need to call the tool first."}269 };270}271 272static void print_diff_split(const std::string & title, const diff_split & diff) {273 LOG_ERR("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET);274 LOG_ERR("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str());275 LOG_ERR("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str());276 LOG_ERR("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str());277 LOG_ERR("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str());278}279 280static void check_reasoning_variables(const common_chat_template & tmpl) {281 LOG_ERR("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET);282 283 try {284 // Create a list of candidate reasoning/thinking variable names to probe285 std::vector<std::string> candidate_vars = {286 "enable_reasoning",287 "use_reasoning",288 "reasoning_enabled",289 "has_reasoning",290 "reasoning_mode",291 "reasoning_format",292 "reasoning_active",293 "with_reasoning",294 "use_thinking",295 "thinking_enabled",296 "has_thinking",297 "thinking_mode",298 "thinking_format",299 "thinking_active",300 "with_thinking",301 "enable_reason",302 "reason_enabled",303 "enable_think",304 "think_enabled",305 };306 307 jinja::context ctx;308 ctx.is_get_stats = true;309 310 json messages = json::array({311 json{312 {"role", "user"},313 {"content", "Test message"}314 },315 json{316 {"role", "assistant"},317 {"content", "Response"},318 {"reasoning_content", "Some reasoning"}319 }320 });321 322 // Set up base context323 jinja::global_from_json(ctx, json{324 {"messages", messages},325 {"tools", json::array()},326 {"bos_token", ""},327 {"eos_token", ""},328 {"add_generation_prompt", false},329 {"enable_thinking", true} // Already passed, so we'll exclude this from results330 }, true);331 332 // Add candidate variables as undefined to probe which ones are accessed333 for (const auto & var_name : candidate_vars) {334 ctx.set_val(var_name, jinja::mk_val<jinja::value_undefined_t>(var_name));335 }336 337 try {338 jinja::runtime runtime(ctx);339 runtime.execute(tmpl.prog);340 } catch (const std::exception & e) {341 // Execution may fail, that's okay - we just want to see what variables were accessed342 }343 344 // Check which candidate variables were accessed (stats.used = true)345 std::vector<std::string> accessed_vars;346 for (const auto & var_name : candidate_vars) {347 auto val = ctx.get_val(var_name);348 if (!val->is_undefined()) {349 // Variable was overwritten, skip it350 continue;351 }352 if (val->stats.used) {353 accessed_vars.push_back(var_name);354 }355 }356 357 if (accessed_vars.empty()) {358 LOG_ERR("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET);359 } else {360 LOG_ERR("Template queries the following reasoning/thinking-related variables:\n");361 for (const auto & var : accessed_vars) {362 LOG_ERR(" %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET);363 }364 }365 366 } catch (const std::exception & e) {367 LOG_ERR("Error checking reasoning variables: %s\n", e.what());368 }369}370 371static void analyze_template(const std::string & template_path) {372 LOG_ERR("\n");373 LOG_ERR("%s", ANSI_PURPLE);374 LOG_ERR("================================================================================\n");375 LOG_ERR(" ANALYZING TEMPLATE: %s\n", template_path.c_str());376 LOG_ERR("================================================================================\n");377 LOG_ERR("%s", ANSI_RESET);378 379 std::string template_source;380 try {381 template_source = read_file(template_path);382 } catch (const std::exception & e) {383 LOG_ERR("Error reading template: %s\n", e.what());384 return;385 }386 387 try {388 common_chat_template chat_template(template_source, "", "");389 json tools = build_tools_definition();390 391 // ===== CAPABILITIES ANALYSIS =====392 LOG_ERR("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET);393 auto caps = chat_template.original_caps();394 LOG_ERR("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false");395 LOG_ERR("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false");396 LOG_ERR("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false");397 LOG_ERR("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false");398 LOG_ERR("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false");399 LOG_ERR("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false");400 401 // ===== DIFFERENTIAL ANALYSIS =====402 403 // Test 1: With and without tools (single user message)404 {405 json user_msg = make_user_msg();406 407 autoparser::generation_params params_no_tools;408 params_no_tools.messages = json::array({ user_msg });409 params_no_tools.add_generation_prompt = false;410 params_no_tools.tools = json::array();411 412 autoparser::generation_params params_with_tools = params_no_tools;413 params_with_tools.tools = tools;414 415 std::string output_no_tools = common_chat_template_direct_apply(chat_template, params_no_tools);416 std::string output_with_tools = common_chat_template_direct_apply(chat_template, params_with_tools);417 418 auto diff = calculate_diff_split(output_no_tools, output_with_tools);419 print_diff_split("Diff: With vs Without Tools (single user message)", diff);420 }421 422 // Test 2: With and without add_generation_prompt (single user message)423 {424 json user_msg = make_user_msg();425 426 autoparser::generation_params params_no_prompt;427 params_no_prompt.messages = json::array({ user_msg });428 params_no_prompt.add_generation_prompt = false;429 params_no_prompt.tools = json::array();430 431 autoparser::generation_params params_with_prompt = params_no_prompt;432 params_with_prompt.add_generation_prompt = true;433 434 std::string output_no_prompt = common_chat_template_direct_apply(chat_template, params_no_prompt);435 std::string output_with_prompt = common_chat_template_direct_apply(chat_template, params_with_prompt);436 437 auto diff = calculate_diff_split(output_no_prompt, output_with_prompt);438 print_diff_split("Diff: With vs Without add_generation_prompt (single user message)", diff);439 }440 441 // Test 3: Assistant with reasoning_content (user, assistant)442 {443 json user_msg = make_user_msg();444 445 autoparser::generation_params params_no_reasoning;446 params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning() });447 params_no_reasoning.add_generation_prompt = false;448 params_no_reasoning.enable_thinking = true;449 450 autoparser::generation_params params_with_reasoning = params_no_reasoning;451 params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning() });452 453 std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);454 std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);455 456 auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);457 print_diff_split("Diff: With vs Without reasoning_content (user, assistant)", diff);458 }459 460 // Test 4: Assistant with reasoning_content (user, assistant, user)461 {462 json user_msg = make_user_msg();463 json user_msg2 = make_user_msg2();464 465 autoparser::generation_params params_no_reasoning;466 params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning(), user_msg2 });467 params_no_reasoning.add_generation_prompt = false;468 params_no_reasoning.enable_thinking = true;469 470 autoparser::generation_params params_with_reasoning = params_no_reasoning;471 params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning(), user_msg2 });472 473 std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);474 std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);475 476 auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);477 print_diff_split("Diff: With vs Without reasoning_content (user, assistant, user)", diff);478 }479 480 // Test 5: Tool call in last assistant message (user, assistant)481 {482 json user_msg = make_user_msg();483 484 autoparser::generation_params params_no_tool;485 params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool() });486 params_no_tool.add_generation_prompt = false;487 params_no_tool.tools = tools;488 489 autoparser::generation_params params_with_tool = params_no_tool;490 params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool() });491 492 std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool);493 std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool);494 495 auto diff = calculate_diff_split(output_no_tool, output_with_tool);496 print_diff_split("Diff: With vs Without tool call (user, assistant)", diff);497 }498 499 // Test 6: Tool call in last assistant message (user, assistant, user)500 {501 json user_msg = make_user_msg();502 json user_msg2 = make_user_msg2_continue();503 504 autoparser::generation_params params_no_tool;505 params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool(), user_msg2 });506 params_no_tool.add_generation_prompt = false;507 params_no_tool.tools = tools;508 509 autoparser::generation_params params_with_tool = params_no_tool;510 params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });511 512 std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool);513 std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool);514 515 auto diff = calculate_diff_split(output_no_tool, output_with_tool);516 print_diff_split("Diff: With vs Without tool call (user, assistant, user)", diff);517 }518 519 // Test 7: One vs two tool calls (user, assistant)520 {521 json user_msg = make_user_msg();522 523 autoparser::generation_params params_one_tool;524 params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool() });525 params_one_tool.add_generation_prompt = false;526 params_one_tool.tools = tools;527 528 autoparser::generation_params params_two_tools = params_one_tool;529 params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools() });530 531 std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool);532 std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools);533 534 auto diff = calculate_diff_split(output_one_tool, output_two_tools);535 print_diff_split("Diff: One vs Two tool calls (user, assistant)", diff);536 }537 538 // Test 8: One vs two tool calls (user, assistant, user)539 {540 json user_msg = make_user_msg();541 json user_msg2 = make_user_msg2_continue();542 543 autoparser::generation_params params_one_tool;544 params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });545 params_one_tool.add_generation_prompt = false;546 params_one_tool.tools = tools;547 548 autoparser::generation_params params_two_tools = params_one_tool;549 params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools(), user_msg2 });550 551 std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool);552 std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools);553 554 auto diff = calculate_diff_split(output_one_tool, output_two_tools);555 print_diff_split("Diff: One vs Two tool calls (user, assistant, user)", diff);556 }557 558 // Test 9: Tool call with vs without reasoning_content (user, assistant)559 {560 json user_msg = make_user_msg();561 562 autoparser::generation_params params_no_reasoning;563 params_no_reasoning.messages = json::array({ user_msg, make_assistant_one_tool() });564 params_no_reasoning.add_generation_prompt = false;565 params_no_reasoning.tools = tools;566 params_no_reasoning.enable_thinking = true;567 568 autoparser::generation_params params_with_reasoning = params_no_reasoning;569 params_with_reasoning.messages = json::array({ user_msg, make_assistant_one_tool_with_reasoning() });570 571 std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning);572 std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning);573 574 auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);575 print_diff_split("Diff: Tool call with vs without reasoning_content (user, assistant)", diff);576 }577 578 // Check reasoning variables579 check_reasoning_variables(chat_template);580 581 } catch (const std::exception & e) {582 LOG_ERR("Analysis failed: %s\n", e.what());583 }584}585 586int main(int argc, char ** argv) {587 // Set log level to capture all output588 common_log_set_verbosity_thold(99);589 590 analysis_options opts;591 if (!parse_options(argc, argv, opts)) {592 return 1;593 }594 595 LOG_ERR("\n");596 LOG_ERR("%s", ANSI_PURPLE);597 LOG_ERR("================================================================================\n");598 LOG_ERR(" TEMPLATE ANALYSIS TOOL\n");599 LOG_ERR("================================================================================\n");600 LOG_ERR("%s", ANSI_RESET);601 LOG_ERR("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET);602 603 for (const auto & path : opts.template_paths) {604 analyze_template(path);605 }606 607 LOG_ERR("\n");608 LOG_ERR("%s", ANSI_GREEN);609 LOG_ERR("================================================================================\n");610 LOG_ERR(" ANALYSIS COMPLETE\n");611 LOG_ERR("================================================================================\n");612 LOG_ERR("%s", ANSI_RESET);613 614 return 0;615}616