Felipe97/llama-cpp-compiled
01.1k
1#include "chat-auto-parser.h"2#include "chat-auto-parser-helpers.h"3#include "chat-peg-parser.h"4#include "chat.h"5#include "common.h"6#include "log.h"7#include "peg-parser.h"8 9#include <algorithm>10#include <cctype>11#include <numeric>12#include <ostream>13#include <sstream>14 15#define ANSI_RESET "\033[0m"16#define ANSI_PURPLE "\033[1m\x1b[38;5;126m"17#define ANSI_ORANGE "\033[1m\x1b[38;5;214m"18#define ANSI_RED "\033[1m\x1b[38;5;196m"19 20using json = common_json;21 22namespace autoparser {23 24static const std::string FUN_FIRST = "FFF_FIRST_FUN_F";25static const std::string FUN_SECOND = "SSS_SECOND_FUN_S";26static const std::string ARG_FIRST = "AA_ARG_FST_AA";27static const std::string ARG_SECOND = "BB_ARG_SND_BB";28static const std::string USER_MSG = "U_USER_MSG Hello END_U";29static const std::string USER_MSG_TWO = "V_USER_MSG Hello END_V";30static const std::string ASSISTANT_MSG = "A_ASST_MSG I can help END_A";31static const std::string THINKING_CONTENT = "REASON_PART I am thinking END_R";32static const std::string CALL_ID_001 = "call00001";33static const std::string CALL_ID_002 = "call00002";34static const std::string CALL_ID_999 = "call99999";35 36static std::vector<std::function<void(const common_chat_template & tmpl, autoparser &)>> workarounds(37 { // Old reasoning Qwen templates - they don't really display reasoning content, but we still want to38 // support reasoning on them39 [](const common_chat_template & tmpl, autoparser & analysis) -> void {40 if (tmpl.src.find("content.split('</think>')") != std::string::npos &&41 tmpl.src.find("reasoning_content") == std::string::npos &&42 tmpl.src.find("<SPECIAL_12>") == std::string::npos &&43 analysis.reasoning.mode == reasoning_mode::NONE) {44 analysis.reasoning.mode = reasoning_mode::TAG_BASED;45 analysis.reasoning.start = "<think>";46 analysis.reasoning.end = "</think>";47 analysis.preserved_tokens.push_back("<think>");48 analysis.preserved_tokens.push_back("</think>");49 LOG_DBG(ANSI_ORANGE "[Patch: old Qwen/Deepseek thinking template]\n" ANSI_RESET);50 }51 },52 // Granite 3.3, with separate reasoning and content markers53 [](const common_chat_template & tmpl, autoparser & analysis) -> void {54 if (tmpl.src.find("Write your thoughts between <think></think> and write your response between "55 "<response></response>") != std::string::npos) {56 analysis.reasoning.mode = reasoning_mode::TAG_BASED;57 analysis.reasoning.start = "<think>";58 analysis.reasoning.end = "</think>";59 analysis.preserved_tokens.push_back("<think>");60 analysis.preserved_tokens.push_back("</think>");61 analysis.content.mode = content_mode::WRAPPED_WITH_REASONING;62 analysis.content.start = "<response>";63 analysis.content.end = "</response>";64 analysis.preserved_tokens.push_back("<response>");65 analysis.preserved_tokens.push_back("</response>");66 LOG_DBG(ANSI_ORANGE "[Patch: Granite 3.3]\n" ANSI_RESET);67 }68 },69 // Cohere Command R+ - content wrapped in <|CHATBOT_TOKEN|>...<|END_OF_TURN_TOKEN|>70 [](const common_chat_template & tmpl, autoparser & analysis) -> void {71 if (tmpl.src.find("<|CHATBOT_TOKEN|>") != std::string::npos &&72 tmpl.src.find("<|END_OF_TURN_TOKEN|>") != std::string::npos && analysis.content.start.empty()) {73 analysis.content.mode = content_mode::ALWAYS_WRAPPED;74 analysis.content.start = "<|CHATBOT_TOKEN|>";75 analysis.content.end = "<|END_OF_TURN_TOKEN|>";76 analysis.preserved_tokens.push_back("<|CHATBOT_TOKEN|>");77 analysis.preserved_tokens.push_back("<|END_OF_TURN_TOKEN|>");78 analysis.user_start = "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>";79 LOG_DBG(ANSI_ORANGE "[Patch: Cohere Command R+]\n" ANSI_RESET);80 }81 },82 // Functionary - no tool call section delimiter83 [](const common_chat_template & tmpl, autoparser & analysis) -> void {84 if (tmpl.src.find("set has_code_interpreter = tools | selectattr(\"type\", \"equalto\", "85 "\"code_interpreter\") | list | length > 0") != std::string::npos) {86 analysis.content.mode = content_mode::PLAIN;87 analysis.content.end = "";88 analysis.tools.function.name_prefix = "";89 analysis.tools.format.section_start = "";90 analysis.tools.format.section_end = "";91 analysis.tools.format.per_call_start = "<function=";92 analysis.tools.format.per_call_end = "</function>";93 analysis.tools.function.close = "";94 analysis.preserved_tokens.clear();95 analysis.preserved_tokens.push_back("<|eot_id|>");96 analysis.preserved_tokens.push_back("<|eom_id|>");97 analysis.preserved_tokens.push_back("<function=");98 analysis.preserved_tokens.push_back(">");99 analysis.preserved_tokens.push_back("</function>");100 LOG_DBG(ANSI_ORANGE "[Patch: Functionary 3.1]\n" ANSI_RESET);101 }102 },103 // DeepSeek-R1-Distill-Qwen104 [](const common_chat_template & tmpl, autoparser & analysis) -> void {105 if (tmpl.src.find(106 "{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>'") !=107 std::string::npos) {108 analysis.tools.format.section_start = "<｜tool▁calls▁begin｜>";109 analysis.tools.format.section_end = "<｜tool▁calls▁end｜>";110 analysis.tools.format.per_call_start = "<｜tool▁call▁begin｜>function";111 analysis.tools.function.name_prefix = "<｜tool▁sep｜>";112 analysis.tools.format.per_call_end = "<｜tool▁call▁end｜>";113 analysis.tools.function.close = "```";114 LOG_DBG(ANSI_ORANGE "[Patch: DeepSeek-R1-Distill-Qwen]\n" ANSI_RESET);115 }116 },117 // Nemotron Nano v2118 [](const common_chat_template & tmpl, autoparser & analysis) -> void {119 if (tmpl.src.find("<SPECIAL_10>") != std::string::npos && tmpl.src.find("<SPECIAL_11>") != std::string::npos &&120 tmpl.src.find("<SPECIAL_12>") != std::string::npos && tmpl.src.find("<TOOL_RESPONSE>") != std::string::npos) {121 122 analysis.tools.format.mode = tool_format::JSON_NATIVE;123 analysis.tools.format.section_start = "";124 analysis.tools.format.section_end = "";125 analysis.tools.format.per_call_start = "<TOOLCALL>";126 analysis.tools.format.per_call_end = "</TOOLCALL>";127 analysis.tools.format.tools_array_wrapped = true;128 analysis.content.mode = content_mode::PLAIN;129 analysis.content.start = "";130 analysis.content.end = "";131 analysis.reasoning.mode = reasoning_mode::TAG_BASED;132 analysis.reasoning.start = "<think>\n";133 analysis.reasoning.end = "</think>";134 analysis.assistant_start = "<SPECIAL_11>Assistant";135 analysis.user_start = "<SPECIAL_11>User";136 analysis.preserved_tokens.clear();137 analysis.preserved_tokens.push_back("<SPECIAL_11>");138 analysis.preserved_tokens.push_back("</think>");139 analysis.preserved_tokens.push_back("<TOOLCALL>");140 analysis.preserved_tokens.push_back("</TOOLCALL>");141 LOG_DBG(ANSI_ORANGE "[Patch: Nemotron Nano v2]\n" ANSI_RESET);142 }143 },144 // Fireworks145 [](const common_chat_template & tmpl, autoparser & analysis) -> void {146 if (tmpl.src.find("{%- set system_prompt = '<|start_header_id|>' + 'system' + '<|end_header_id|>\\n\\n'"147 " + message['content'] | trim + '\\n' + system_prompt_suffix + '<|eot_id|>' -%}") != std::string::npos) {148 analysis.assistant_start = "<|start_header_id|>assistant<|end_header_id|>";149 analysis.user_start = "<|start_header_id|>user<|end_header_id|>";150 LOG_DBG(ANSI_ORANGE "[Patch: Fireworks v2]\n" ANSI_RESET);151 }152 },153 // Solar Open154 [](const common_chat_template & tmpl, autoparser & analysis) -> void {155 if (tmpl.src.find("<|begin|>assistant<|think|><|end|>") != std::string::npos) {156 analysis.assistant_start = "<|begin|>assistant";157 LOG_DBG(ANSI_ORANGE "[Patch: Solar Open]\n" ANSI_RESET);158 }159 },160 // Apriel 1.6161 [](const common_chat_template & tmpl, autoparser & analysis) -> void {162 if (tmpl.src.find("if not loop.last and '[BEGIN FINAL RESPONSE]' in asst_text") != std::string::npos) {163 analysis.user_start = "<|begin_user|>";164 analysis.assistant_start = "<|begin_assistant|>";165 LOG_DBG(ANSI_ORANGE "[Patch: Apriel 1.6]\n" ANSI_RESET);166 }167 },168 // template uses the JSON {name, parameters} tool instruction, emits the OpenAI function wrapper169 [](const common_chat_template & tmpl, autoparser & analysis) -> void {170 if (tmpl.src.find("Respond in the format {\"name\": function name") != std::string::npos &&171 tmpl.src.find("Do not use variables.") != std::string::npos) {172 analysis.tools.format.openai_wrapper_trigger = true;173 LOG_DBG(ANSI_ORANGE "[Patch: JSON name/parameters tool instruction]\n" ANSI_RESET);174 }175 },176 // Laguna (poolside) - the v4 chat template renders reasoning and tool-arg177 // delimiters with formatting whitespace ("<think>\n", "</arg_value>\n") that178 // the model does not emit, so the inferred delimiters carry a spurious179 // newline and never match the model output. Trim to the bare tag. (v8180 // renders without the whitespace, so this is a no-op there.)181 [](const common_chat_template & tmpl, autoparser & analysis) -> void {182 if (tmpl.src.find("laguna_glm_thinking") != std::string::npos) {183 analysis.reasoning.start = trim_whitespace(analysis.reasoning.start);184 analysis.reasoning.end = trim_whitespace(analysis.reasoning.end);185 analysis.tools.arguments.value_prefix = trim_whitespace(analysis.tools.arguments.value_prefix);186 analysis.tools.arguments.value_suffix = trim_whitespace(analysis.tools.arguments.value_suffix);187 analysis.tools.arguments.separator = trim_whitespace(analysis.tools.arguments.separator);188 analysis.tools.arguments.tolerate_intertag_whitespace = true;189 // The CONTROL/eot </assistant> token only halts generation when emitted as the190 // single token; after tool calls the model can spell it out as text tokens.191 // A literal stop string catches it either way.192 analysis.additional_stops.push_back("</assistant>");193 LOG_DBG(ANSI_ORANGE "[Patch: Laguna]\n" ANSI_RESET);194 }195 },196 // Bailing V3197 [](const common_chat_template & tmpl, autoparser & analysis) -> void {198 if (tmpl.src.find("Bailing V3 chat template") != std::string::npos) {199 analysis.tools.arguments.value_suffix = trim_whitespace(analysis.tools.arguments.value_suffix);200 analysis.tools.arguments.tolerate_intertag_whitespace = true;201 LOG_DBG(ANSI_ORANGE "[Patch: Bailing V3]\n" ANSI_RESET);202 }203 },204 205 });206 207// Common JSON structures208static json params_schema = {209 { "type", "object" },210 { "properties",211 { { ARG_FIRST, { { "type", "string" }, { "description", "First argument" } } },212 { ARG_SECOND, { { "type", "string" }, { "description", "Second argument" } } } } },213 { "required", json::array({}) }214};215 216static json tools = json::array({217 { { "type", "function" },218 { "function",219 json{ { "name", FUN_FIRST }, { "description", "Test function foo" }, { "parameters", params_schema } } } },220 { { "type", "function" },221 { "function",222 json{ { "name", FUN_SECOND }, { "description", "Test function bar" }, { "parameters", params_schema } } } }223});224 225static json user_msg = json{226 { "role", "user" },227 { "content", USER_MSG }228};229 230static json build_tool_call(const std::string & name, const json & args, const std::string & id = CALL_ID_001) {231 return json{232 { "id", id },233 { "type", "function" },234 { "function", json{ { "name", name }, { "arguments", args } } }235 };236}237 238static json first_tool_call_zero_args = build_tool_call(FUN_FIRST, json::object(), CALL_ID_001);239static json first_tool_call_one_arg = build_tool_call(FUN_FIRST, {{ ARG_FIRST, "XXXX" }}, CALL_ID_001);240static json first_tool_call_one_arg_other_val = build_tool_call(FUN_FIRST, {{ ARG_FIRST, "YYYY" }}, CALL_ID_001);241static json first_tool_call_other_arg = build_tool_call(FUN_FIRST, {{ ARG_SECOND, "YYYY" }}, CALL_ID_001);242 243static json first_tool_call =244 build_tool_call(FUN_FIRST, json{{ ARG_FIRST, "XXXX" }, { ARG_SECOND, "YYYY" }}, CALL_ID_001);245static json second_tool_call =246 build_tool_call(FUN_SECOND, json{ { ARG_FIRST, "XXXX" }, { ARG_SECOND, "YYYY" }}, CALL_ID_002);247static json first_tool_call_alt_id =248 build_tool_call(FUN_FIRST, json{{ ARG_FIRST, "XXXX" }, { ARG_SECOND, "YYYY" }}, CALL_ID_999);249 250template <typename T>251static std::string mode_to_str(T mode) {252 std::ostringstream os;253 os << mode;254 return os.str();255}256 257void autoparser::analyze_template(const common_chat_template & tmpl) {258 jinja_caps = tmpl.original_caps();259 reasoning = analyze_reasoning(tmpl, jinja_caps.supports_tool_calls);260 content = analyze_content(tmpl, reasoning);261 tools = analyze_tools(jinja_caps.supports_tool_calls ? analyze_tools(tmpl, jinja_caps, reasoning) : analyze_tools());262 assistant_start = detect_assistant_start_marker(tmpl);263 user_start = detect_user_start_marker(tmpl);264 collect_preserved_tokens();265 266 for (auto & workaround : workarounds) {267 workaround(tmpl, *this);268 }269 270 LOG_DBG("\n--- Reasoning & Content Structure ---\n");271 LOG_DBG("user_msg_start: %s\n", user_start.c_str());272 LOG_DBG("assistant_msg_start: %s\n", assistant_start.c_str());273 LOG_DBG("reasoning_mode: %s\n", mode_to_str(reasoning.mode).c_str());274 LOG_DBG("reasoning_start: '%s'\n", reasoning.start.c_str());275 LOG_DBG("reasoning_end: '%s'\n", reasoning.end.c_str());276 LOG_DBG("content_mode: %s\n", mode_to_str(content.mode).c_str());277 LOG_DBG("content_start: '%s'\n", content.start.c_str());278 LOG_DBG("content_end: '%s'\n", content.end.c_str());279 280 LOG_DBG("\n--- Tool Call Structure ---\n");281 LOG_DBG("tool_mode: %s\n", mode_to_str(tools.format.mode).c_str());282 LOG_DBG("supports_tools: %s\n", jinja_caps.supports_tools ? "true" : "false");283 LOG_DBG("supports_parallel_calls: %s\n", jinja_caps.supports_parallel_tool_calls ? "true" : "false");284 LOG_DBG("tool_section_start: '%s'\n", tools.format.section_start.c_str());285 LOG_DBG("tool_section_end: '%s'\n", tools.format.section_end.c_str());286 LOG_DBG("per_call_start: '%s'\n", tools.format.per_call_start.c_str());287 LOG_DBG("per_call_end: '%s'\n", tools.format.per_call_end.c_str());288 LOG_DBG("func_name_prefix: '%s'\n", tools.function.name_prefix.c_str());289 LOG_DBG("func_name_suffix: '%s'\n", tools.function.name_suffix.c_str());290 LOG_DBG("func_args_separator: '%s'\n", tools.function.args_separator.c_str());291 LOG_DBG("func_close: '%s'\n", tools.function.close.c_str());292 LOG_DBG("call_id_prefix: '%s'\n", tools.call_id.prefix.c_str());293 LOG_DBG("call_id_suffix: '%s'\n", tools.call_id.suffix.c_str());294 LOG_DBG("call_id_pos: '%s'\n", mode_to_str(tools.call_id.pos).c_str());295 LOG_DBG("args_start: '%s'\n", tools.arguments.start.c_str());296 LOG_DBG("args_end: '%s'\n", tools.arguments.end.c_str());297 LOG_DBG("arg_name_prefix: '%s'\n", tools.arguments.name_prefix.c_str());298 LOG_DBG("arg_name_suffix: '%s'\n", tools.arguments.name_suffix.c_str());299 LOG_DBG("arg_value_prefix: '%s'\n", tools.arguments.value_prefix.c_str());300 LOG_DBG("arg_value_suffix: '%s'\n", tools.arguments.value_suffix.c_str());301 LOG_DBG("name_field: '%s'\n", tools.format.name_field.c_str());302 LOG_DBG("args_field: '%s'\n", tools.format.args_field.c_str());303 LOG_DBG("id_field: '%s'\n", tools.format.id_field.c_str());304 LOG_DBG("gen_id_field: '%s'\n", tools.format.gen_id_field.c_str());305 LOG_DBG("parameter_order: '%s'\n", std::accumulate(tools.format.parameter_order.begin(), tools.format.parameter_order.end(),306 std::string(""), [] (const std::string & a, const std::string & b) { return a.empty() ? b : a + ", " + b; }307 ).c_str());308 309 LOG_DBG(ANSI_PURPLE "=== Differential analysis complete ===\n" ANSI_RESET);310 analysis_complete = true;311}312 313void autoparser::collect_preserved_tokens() {314 auto add_token = [this](const std::string & org_token) {315 std::string token = trim_whitespace(org_token);316 if (!token.empty()) {317 // Avoid duplicates318 if (std::find(preserved_tokens.begin(), preserved_tokens.end(), token) == preserved_tokens.end()) {319 preserved_tokens.push_back(token);320 }321 }322 };323 324 add_token(reasoning.start);325 add_token(reasoning.end);326 add_token(content.start);327 add_token(content.end);328 add_token(tools.format.section_start);329 add_token(tools.format.section_end);330 add_token(tools.format.per_call_start);331 add_token(tools.format.per_call_end);332 add_token(tools.function.name_prefix);333 add_token(tools.function.name_suffix);334 add_token(tools.function.args_separator);335 add_token(tools.function.close);336 add_token(tools.arguments.start);337 add_token(tools.arguments.end);338 add_token(tools.arguments.name_prefix);339 add_token(tools.arguments.name_suffix);340 add_token(tools.arguments.separator);341 add_token(tools.arguments.value_prefix);342 add_token(tools.arguments.value_suffix);343 add_token(tools.call_id.prefix);344 add_token(tools.call_id.suffix);345}346 347std::string autoparser::detect_assistant_start_marker(const common_chat_template & tmpl) {348 json user_msg = json{349 { "role", "user" },350 { "content", USER_MSG }351 };352 353 json assistant_no_reasoning = json{354 { "role", "assistant" },355 { "content", ASSISTANT_MSG }356 };357 358 template_params params;359 params.messages = json::array({ user_msg });360 params.add_generation_prompt = false;361 params.enable_thinking = true;362 363 auto comparison = compare_variants(364 tmpl, params, [&](template_params & p) {365 p.messages = json::array({ user_msg, assistant_no_reasoning });366 }367 );368 369 if (!comparison) {370 LOG_DBG(ANSI_ORANGE "%s: Template application failed, skipping assistant start detection\n" ANSI_RESET, __func__);371 return "";372 }373 374 auto usermsg = comparison->diff.right;375 if (usermsg.find(ASSISTANT_MSG) == std::string::npos) {376 LOG_DBG(ANSI_ORANGE "%s: Did not find assistant message in assistant message block, skipping detection\n" ANSI_RESET, __func__);377 }378 379 auto ast_prefix = usermsg.substr(0, usermsg.find(ASSISTANT_MSG));380 if (!reasoning.start.empty() && ast_prefix.find(trim_whitespace(reasoning.start)) != std::string::npos) {381 ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.start)));382 }383 if (!reasoning.end.empty() && ast_prefix.find(trim_whitespace(reasoning.end)) != std::string::npos) {384 ast_prefix = ast_prefix.substr(0, ast_prefix.find(trim_whitespace(reasoning.end)));385 }386 return trim_whitespace(ast_prefix);387}388 389std::string autoparser::detect_user_start_marker(const common_chat_template & tmpl) {390 json user_msg = json{391 { "role", "user" },392 { "content", USER_MSG }393 };394 395 json assistant = json{396 { "role", "assistant" },397 { "content", ASSISTANT_MSG }398 };399 400 json user_msg_two = json{401 { "role", "user" },402 { "content", USER_MSG_TWO }403 };404 405 template_params params;406 params.messages = json::array({});407 params.add_generation_prompt = false;408 params.enable_thinking = true;409 410 auto comparison = compare_variants(411 tmpl, params, [&](template_params & p) {412 p.messages = json::array({ user_msg });413 }414 );415 416 if (!comparison) {417 LOG_DBG(ANSI_ORANGE "%s: Template application failed, unsupported empty messages? trying complex variant\n" ANSI_RESET, __func__);418 params.messages = json::array({ user_msg_two, assistant });419 comparison = compare_variants(420 tmpl, params, [&](template_params & p) {421 p.messages = json::array({ user_msg_two, assistant, user_msg });422 }423 );424 if (!comparison) {425 LOG_DBG(ANSI_ORANGE "%s: Template application failed for reserve variant, aborting\n" ANSI_RESET, __func__);426 return "";427 }428 }429 430 auto usermsg = comparison->diff.right;431 if (usermsg.find(USER_MSG) == std::string::npos) {432 LOG_DBG(ANSI_ORANGE "%s: Did not find user message in user message block, aborting detection\n" ANSI_RESET, __func__);433 }434 435 if (usermsg.find(ASSISTANT_MSG) != std::string::npos) {436 usermsg = usermsg.substr(usermsg.find(ASSISTANT_MSG) + ASSISTANT_MSG.size());437 }438 439 auto candidate = usermsg.substr(0, usermsg.find(USER_MSG));440 auto candidate_split = segmentize_markers(candidate);441 std::stringstream result;442 bool encountered_marker = false;443 for (const auto & mrk : candidate_split) {444 std::string lower_mrk = std::string(mrk.value);445 std::transform(lower_mrk.begin(), lower_mrk.end(), lower_mrk.begin(),446 [](unsigned char c) { return std::tolower(c); });447 // heuristic to weed out potential end markers, but only at the start448 if (mrk.type == segment_type::MARKER && !encountered_marker &&449 (lower_mrk.find("end") != std::string::npos || lower_mrk.find("close") != std::string::npos)) {450 continue;451 }452 if (mrk.type == segment_type::TEXT && !encountered_marker && trim_whitespace(mrk.value).empty()) {453 continue;454 }455 encountered_marker |= mrk.type == segment_type::MARKER;456 result << mrk.value;457 }458 return trim_whitespace(result.str());459}460 461analyze_reasoning::analyze_reasoning(const common_chat_template & tmpl, bool supports_tools)462 : analyze_base(tmpl) {463 LOG_DBG(ANSI_PURPLE "=== Starting differential analysis ===\n" ANSI_RESET);464 LOG_DBG(ANSI_ORANGE "Phase 1: Reasoning analysis\n" ANSI_RESET);465 466 compare_reasoning_presence();467 compare_thinking_enabled();468 if (supports_tools) {469 compare_reasoning_scope();470 }471}472 473void analyze_reasoning::compare_reasoning_presence() {474 json user_msg = json{475 { "role", "user" },476 { "content", USER_MSG }477 };478 479 json assistant_no_reasoning = json{480 { "role", "assistant" },481 { "content", ASSISTANT_MSG }482 };483 484 json assistant_with_reasoning = json{485 { "role", "assistant" },486 { "content", ASSISTANT_MSG },487 { "reasoning_content", THINKING_CONTENT }488 };489 490 template_params params;491 params.messages = json::array({ user_msg, assistant_no_reasoning });492 params.add_generation_prompt = false;493 params.enable_thinking = true;494 495 auto comparison = compare_variants(496 *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_reasoning }); });497 498 if (!comparison) {499 LOG_DBG(ANSI_ORANGE "%s: Template application failed, skipping reasoning detection\n" ANSI_RESET, __func__);500 return;501 }502 503 const auto & diff = comparison->diff;504 505 const std::string reasoning_content = THINKING_CONTENT;506 507 if (!diff.right.empty() && diff.right.find(reasoning_content) != std::string::npos) {508 auto parser_delimiter = build_tagged_peg_parser([&](common_peg_parser_builder &p) {509 return p.literal(reasoning_content) + p.space() + p.optional(p.tag("post", (p.marker() + p.space())) + p.rest());510 });511 auto parser_wrapped = build_tagged_peg_parser([&](common_peg_parser_builder &p) {512 return p.tag("pre", p.marker() + p.space()) + p.literal(reasoning_content) + p.tag("post", (p.space() + p.marker() + p.space())) + p.rest();513 });514 // try the more aggressive parse first, if it fails, fall back to the delimiter one515 auto result = parser_wrapped.parse_anywhere_and_extract(comparison->output_B);516 if (!result.result.success()) {517 result = parser_delimiter.parse_anywhere_and_extract(comparison->output_B);518 }519 if (result.result.success()) {520 if (!result.tags["pre"].empty() && !result.tags["post"].empty()) {521 mode = reasoning_mode::TAG_BASED;522 start = result.tags["pre"];523 end = result.tags["post"];524 } else if (!result.tags["post"].empty()) {525 mode = reasoning_mode::TAG_BASED;526 end = result.tags["post"];527 }528 }529 }530}531 532void analyze_reasoning::compare_thinking_enabled() {533 json user_msg = json{534 { "role", "user" },535 { "content", USER_MSG }536 };537 538 template_params params;539 params.messages = json::array({ user_msg });540 params.add_generation_prompt = true;541 params.enable_thinking = false;542 543 auto comparison = compare_variants(*tmpl, params, [&](template_params & p) { p.enable_thinking = true; });544 545 if (!comparison) {546 LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET , __func__);547 return;548 }549 550 const auto & diff = comparison->diff;551 552 std::string left_trimmed = trim_whitespace(diff.left);553 std::string right_trimmed = trim_whitespace(diff.right);554 555 if (left_trimmed.empty() && !diff.right.empty()) {556 if (!right_trimmed.empty() && string_ends_with(comparison->output_B, right_trimmed)) {557 if (start.empty()) {558 start = diff.right;559 mode = reasoning_mode::TAG_BASED;560 }561 }562 } else if (right_trimmed.empty() && !diff.left.empty()) {563 if (!left_trimmed.empty() && string_ends_with(comparison->output_A, left_trimmed)) {564 if (end.empty()) {565 auto seg = prune_whitespace_segments(segmentize_markers(comparison->output_A));566 if (seg.size() >= 2 && seg[seg.size() - 1].value == left_trimmed && seg[seg.size() - 2].type == segment_type::MARKER) {567 start = seg[seg.size() - 2].value;568 }569 end = diff.left;570 mode = reasoning_mode::TAG_BASED;571 }572 }573 } else if (!left_trimmed.empty() && !right_trimmed.empty()) {574 // Full-output diff is noisy (e.g., SmolLM3 changes the system message when enable_thinking flips).575 // Try to find reasoning markers by tail-anchoring:576 // one output's generation prompt tail may appear in the other with extra reasoning markers appended.577 const auto & output_A = comparison->output_A;578 const auto & output_B = comparison->output_B;579 const size_t anchor_len = 64;580 581 for (int dir = 0; dir < 2; dir++) {582 const auto & base = dir == 0 ? output_B : output_A;583 const auto & extended = dir == 0 ? output_A : output_B;584 585 size_t len = std::min(base.size(), anchor_len);586 std::string anchor = base.substr(base.size() - len);587 auto pos = extended.rfind(anchor);588 if (pos == std::string::npos || pos + len >= extended.size()) {589 continue;590 }591 592 std::string extra = trim_whitespace(extended.substr(pos + len));593 if (extra.empty()) {594 continue;595 }596 597 auto seg = prune_whitespace_segments(segmentize_markers(extra));598 if (seg.size() == 2 && seg[0].type == segment_type::MARKER && seg[1].type == segment_type::MARKER) {599 if (start.empty()) {600 start = seg[0].value;601 }602 if (end.empty()) {603 end = seg[1].value;604 }605 mode = reasoning_mode::TAG_BASED;606 break;607 }608 }609 }610 611 if (mode == reasoning_mode::NONE && start.empty() && !end.empty()) {612 mode = reasoning_mode::TAG_BASED;613 }614}615 616void analyze_reasoning::compare_reasoning_scope() {617 json assistant_reasoning_content = json{618 { "role", "assistant" },619 { "content", ASSISTANT_MSG },620 { "reasoning_content", THINKING_CONTENT }621 };622 623 json assistant_reasoning_tools = json{624 { "role", "assistant" },625 { "content", nullptr },626 { "reasoning_content", THINKING_CONTENT },627 { "tool_calls",628 json::array({ build_tool_call(FUN_FIRST, json{ { ARG_FIRST, "VVVV" }, { ARG_SECOND, "XXXX" } }) }) }629 };630 631 template_params params;632 params.messages = json::array({ user_msg, assistant_reasoning_content });633 params.tools = tools;634 params.add_generation_prompt = false;635 params.enable_thinking = true;636 637 auto comparison = compare_variants(638 *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_reasoning_tools }); });639 640 if (!comparison) {641 LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__);642 return;643 }644 645 std::string reasoning_content = THINKING_CONTENT;646 647 // Check if reasoning only appears in variant B (with tools)648 bool reasoning_in_A = comparison->output_A.find(reasoning_content) != std::string::npos;649 bool reasoning_in_B = comparison->output_B.find(reasoning_content) != std::string::npos;650 651 if (!reasoning_in_A && reasoning_in_B) {652 mode = reasoning_mode::TOOLS_ONLY;653 LOG_DBG(ANSI_ORANGE "%s: Detected TOOLS_ONLY reasoning mode\n" ANSI_RESET, __func__);654 655 auto parser_wrapped = build_tagged_peg_parser([&](common_peg_parser_builder &p) {656 return p.tag("pre", p.marker() + p.space()) + p.literal(reasoning_content) + p.space() + p.tag("post", (p.marker() + p.space()));657 });658 auto result = parser_wrapped.parse_anywhere_and_extract(comparison->output_B);659 if (result.result.success()) {660 start = result.tags["pre"];661 end = result.tags["post"];662 } else {663 auto parser_delimiter = build_tagged_peg_parser([&](common_peg_parser_builder &p) {664 return p.literal(reasoning_content) + p.space() + p.optional(p.tag("post", (p.marker() + p.space())));665 });666 result = parser_delimiter.parse_anywhere_and_extract(comparison->output_B);667 if (result.result.success()) {668 end = result.tags["post"];669 } else {670 LOG_DBG(ANSI_ORANGE "%s: Unable to extract reasoning markers, falling back to reasoning = NONE\n" ANSI_RESET, __func__);671 mode = reasoning_mode::NONE;672 }673 }674 }675}676 677analyze_content::analyze_content(const common_chat_template & tmpl, const analyze_reasoning & reasoning)678 : analyze_base(tmpl) {679 LOG_DBG(ANSI_ORANGE "Phase 2: Content analysis\n" ANSI_RESET);680 681 json assistant_content_only = json{682 { "role", "assistant" },683 { "content", ASSISTANT_MSG }684 };685 686 json assistant_with_tools = json{687 { "role", "assistant" },688 { "content", "" },689 { "tool_calls", json::array({ build_tool_call("test_func", json{ { "arg1", "value1" } }) }) }690 };691 692 json assistant_with_reasoning = json{693 { "role", "assistant" },694 { "content", "" },695 { "reasoning_content", THINKING_CONTENT }696 };697 698 template_params params_content_only;699 params_content_only.messages = json::array({ user_msg, assistant_content_only });700 params_content_only.add_generation_prompt = false;701 params_content_only.enable_thinking = true;702 params_content_only.tools = tools;703 704 auto comparison_with_tools = compare_variants(tmpl, params_content_only, [&](template_params & p) {705 p.messages = json::array({ user_msg, assistant_with_tools });706 });707 708 auto comparison_with_reasoning = compare_variants(tmpl, params_content_only, [&](template_params & p) {709 p.messages = json::array({ user_msg, assistant_with_reasoning });710 });711 712 if (!comparison_with_tools || !comparison_with_reasoning) {713 LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__);714 return;715 }716 717 const auto & diff_tools = comparison_with_tools->diff;718 const auto & diff_reasoning = comparison_with_reasoning->diff;719 720 std::string response = ASSISTANT_MSG;721 722 bool found_plain_content = false;723 if (trim_whitespace(diff_tools.left) == response) {724 auto parser = build_tagged_peg_parser([&](common_peg_parser_builder & p) {725 return p.space() + diff_reasoning.left + p.space() + p.optional(p.marker()) + p.space() + p.end();726 });727 if (parser.parse_and_extract(diff_reasoning.left).result.success()) {728 // We only have the content text in the diff (possibly with a stray EOG marker), so no markers729 mode = content_mode::PLAIN;730 found_plain_content = true;731 } else if (reasoning.mode != reasoning_mode::NONE && !reasoning.end.empty()) {732 auto post_reasoning_parser = build_tagged_peg_parser([&](common_peg_parser_builder & p) {733 return p.literal(reasoning.end) + p.space() + p.literal(response);734 });735 if (post_reasoning_parser.parse_anywhere_and_extract(diff_reasoning.left).result.success()) {736 mode = content_mode::PLAIN;737 found_plain_content = true;738 }739 }740 }741 if (!found_plain_content) {742 std::string rdiff = diff_reasoning.left;743 if (!reasoning.end.empty() && rdiff.find(reasoning.end) != std::string::npos) {744 rdiff = rdiff.substr(rdiff.find(reasoning.end) + reasoning.end.length());745 }746 // Take the more promising diff747 std::string pure_content = rdiff.length() > diff_tools.left.length() ? rdiff : diff_tools.left;748 auto parser_wrapped = build_tagged_peg_parser([&](common_peg_parser_builder &p) {749 return p.tag("pre", p.marker() + p.space()) + p.literal(response) + p.space() + p.tag("post", (p.marker() + p.space())) + p.rest();750 });751 auto result = parser_wrapped.parse_anywhere_and_extract(pure_content);752 start = result.tags["pre"];753 end = result.tags["post"];754 // TODO: WRAPPED_WITH_REASONING755 }756 757 // Determine content mode758 if (!start.empty() || !end.empty()) {759 mode = content_mode::ALWAYS_WRAPPED;760 // TODO: END_DELIMITED content mode - delimited at end but not at start?761 }762}763 764bool analyze_content::is_always_wrapped() const {765 return mode == content_mode::ALWAYS_WRAPPED && !start.empty() && !end.empty();766}767 768analyze_tools::analyze_tools(const common_chat_template & tmpl,769 const jinja::caps & caps,770 const analyze_reasoning & reasoning)771 : analyze_base(tmpl) {772 LOG_DBG(ANSI_ORANGE "Phase 3: Tool call analysis\n" ANSI_RESET);773 774 analyze_tool_calls(reasoning, caps.supports_parallel_tool_calls);775 776 if (format.mode != tool_format::NONE && format.mode != tool_format::JSON_NATIVE) {777 if (caps.supports_parallel_tool_calls) {778 check_per_call_markers();779 }780 LOG_DBG(ANSI_ORANGE "Phase 3a: Function call analysis\n" ANSI_RESET);781 extract_function_markers();782 LOG_DBG(ANSI_ORANGE "Phase 3b: Argument analysis\n" ANSI_RESET);783 if (format.mode == tool_format::TAG_WITH_TAGGED) {784 analyze_arguments();785 }786 extract_argument_separator();787 extract_args_markers();788 LOG_DBG(ANSI_ORANGE "Phase 3c: Call id analysis\n" ANSI_RESET);789 extract_call_id_markers();790 }791}792 793void analyze_tools::analyze_tool_calls(const analyze_reasoning & reasoning, bool supports_parallel_tool_calls) {794 json assistant_no_tools = json{795 { "role", "assistant" },796 { "content", ASSISTANT_MSG }797 };798 799 json assistant_with_tools = json{800 { "role", "assistant" },801 { "content", "" },802 { "tool_calls", json::array({ first_tool_call }) }803 };804 805 template_params params;806 params.messages = json::array({ user_msg, assistant_no_tools });807 params.tools = tools;808 params.add_generation_prompt = false;809 params.enable_thinking = true;810 811 auto comparison = compare_variants(812 *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_with_tools }); });813 814 if (!comparison) {815 LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__);816 return;817 }818 819 const auto & diff = comparison->diff;820 821 std::string tool_section = diff.right;822 823 if (tool_section.empty()) {824 return;825 }826 827 analyze_tool_call_format(tool_section, FUN_FIRST, ARG_FIRST, reasoning, supports_parallel_tool_calls);828}829 830void analyze_tools::analyze_tool_call_format(const std::string & haystack,831 const std::string & fun_name_needle,832 const std::string & arg_name_needle,833 const analyze_reasoning & reasoning,834 bool supports_parallel_tool_calls) {835 if (fun_name_needle.empty() || arg_name_needle.empty() || haystack.empty()) {836 return;837 }838 839 auto in_json_haystack = [&haystack](const std::string & needle) -> bool {840 auto parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {841 return p.choice({ p.literal("{"), p.literal(":") }) << p.choice({842 p.tag("dq", p.literal("\"") + p.literal(needle) + p.literal("\"")) });843 });844 auto result = parser.parse_anywhere_and_extract(haystack);845 return result.result.success();846 };847 848 auto fun_quote = in_json_haystack(fun_name_needle);849 auto arg_quote = in_json_haystack(arg_name_needle);850 851 if (fun_quote) {852 // no need to check further, we're in JSON land853 format.mode = tool_format::JSON_NATIVE;854 } else if (arg_quote) {855 format.mode = tool_format::TAG_WITH_JSON;856 } else {857 format.mode = tool_format::TAG_WITH_TAGGED;858 }859 860 // first, remove any reasoning markers861 std::string clean_haystack = haystack;862 if (!reasoning.start.empty()) {863 auto pos = haystack.find(reasoning.start);864 if (pos != std::string::npos) {865 clean_haystack = haystack.substr(0, pos) + haystack.substr(pos + reasoning.start.length());866 }867 }868 if (!reasoning.end.empty()) {869 auto pos = clean_haystack.find(reasoning.end);870 if (pos != std::string::npos) {871 clean_haystack = clean_haystack.substr(0, pos) + clean_haystack.substr(pos + reasoning.end.length());872 }873 }874 875 if (format.mode == tool_format::JSON_NATIVE) {876 analyze_tool_call_format_json_native(clean_haystack, fun_name_needle, arg_name_needle);877 if (supports_parallel_tool_calls) {878 analyze_json_native_parallel_calls();879 }880 } else {881 analyze_tool_call_format_non_json(clean_haystack, fun_name_needle);882 }883 // always relax whitespace requirements on ending markers since they don't influence content884 format.section_end = trim_whitespace(format.section_end);885 format.per_call_end = trim_whitespace(format.per_call_end);886}887 888void analyze_tools::analyze_json_native_parallel_calls() {889 json assistant_one_tool = json{890 { "role", "assistant" },891 { "content", "" },892 { "tool_calls", json::array({ first_tool_call }) }893 };894 895 json assistant_two_tools = json{896 { "role", "assistant" },897 { "content", "" },898 { "tool_calls", json::array({ first_tool_call, second_tool_call }) }899 };900 901 template_params params;902 params.messages = json::array({ user_msg, assistant_one_tool });903 params.tools = tools;904 params.add_generation_prompt = false;905 params.enable_thinking = true;906 907 auto comparison = compare_variants(908 *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_tools }); });909 910 if (!comparison) {911 LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__);912 return;913 }914 915 std::string & second_call = comparison->diff.right;916 if (!format.section_start.empty() && second_call.find(format.section_start) != std::string::npos) {917 format.per_call_start = format.section_start;918 format.per_call_end = format.section_end;919 format.section_start.clear();920 format.section_end.clear();921 }922}923 924void analyze_tools::analyze_tool_call_format_json_native(const std::string & clean_haystack,925 const std::string & fun_name_needle,926 const std::string & arg_name_needle) {927 // we might not have the typical OpenAI tool calling structure928 int json_start = clean_haystack.find_first_of('{');929 int json_end = clean_haystack.find_last_of('}');930 std::string cut = clean_haystack.substr(json_start, json_end - json_start + 1);931 json call_struct = json::parse(cut);932 auto register_field = [&](const std::string & prefix, const common_json_entry & subel) {933 if (subel.value().is_string() && std::string(subel.value()).find("call0000") != std::string::npos) {934 format.id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key();935 } else if (subel.value().is_string() && std::string(subel.value()) == fun_name_needle) {936 format.name_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key();937 } else if (subel.value().dump().find(arg_name_needle) !=938 std::string::npos) { // handle both string and JSON obj variants939 format.args_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key();940 } else if (subel.key().find("id") != std::string::npos) {941 // heuristics for generated id field942 format.gen_id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key();943 }944 };945 for (const auto & el : call_struct.items()) {946 if (el.key() == fun_name_needle) {947 format.fun_name_is_key = true;948 // When function name is the key, there's no name field and args are direct949 format.name_field.clear();950 format.args_field.clear();951 // Don't register this element - the function name IS the key, not a field952 } else {953 if (el.value().is_object() &&954 el.value().dump().find(arg_name_needle) == std::string::npos) { // not the args object955 format.function_field = el.key();956 for (const auto & subel : el.value().items()) {957 register_field(el.key(), subel);958 }959 }960 // Register this element as a potential field961 register_field("", el);962 }963 }964 auto array_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {965 return p.tag("pre", p.literal("[") + p.space()) + p.literal(cut) + p.tag("post", p.space() + p.literal("]"));966 });967 968 auto ar_parse_res = array_parser.parse_anywhere_and_extract(clean_haystack);969 if (ar_parse_res.result.success()) {970 format.tools_array_wrapped = true;971 json_start -= ar_parse_res.tags["pre"].length();972 json_end += ar_parse_res.tags["post"].length();973 }974 json_end++; // we want to move past the closing char for end marker extraction975 976 std::vector<std::pair<size_t, std::string>> located_params;977 if (!format.name_field.empty()) {978 located_params.push_back({ clean_haystack.find(format.name_field), format.name_field });979 }980 if (!format.args_field.empty()) {981 located_params.push_back({ clean_haystack.find(format.args_field), format.args_field });982 }983 if (!format.id_field.empty()) {984 located_params.push_back({ clean_haystack.find(format.id_field), format.id_field });985 }986 if (!format.gen_id_field.empty()) {987 located_params.push_back({ clean_haystack.find(format.gen_id_field), format.gen_id_field });988 }989 std::sort(located_params.begin(), located_params.end());990 for (auto & pair : located_params) {991 format.parameter_order.push_back(pair.second);992 }993 // we can immediately extract tool calling markers too994 format.section_start = trim_leading_whitespace(clean_haystack.substr(0, json_start));995 format.section_end = trim_whitespace(clean_haystack.substr(json_end));996 // When tools_array_wrapped is true, the closing bracket is part of the array structure,997 // not a separate section end marker. Clear tool_section_end to avoid duplicate brackets.998 if (format.tools_array_wrapped && format.section_end == "]") {999 format.section_end.clear();1000 }1001}1002 1003void analyze_tools::analyze_tool_call_format_non_json(const std::string & clean_haystack,1004 const std::string & fun_name_needle) {1005 // first, let's find out if the function is inside a tag or standalone1006 auto fun_marker_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1007 return p.tag("fun_marker", p.choice({1008 p.tag("fun_pre", p.literal("<") + p.until_one_of({ ">", fun_name_needle })) + p.literal(fun_name_needle) +1009 p.tag("fun_post", p.negate(p.space() + p.literal("<")) + p.until(">") + p.literal(">")) + p.space(),1010 p.tag("fun_pre", p.literal("[") + p.until_one_of({ "]", fun_name_needle })) + p.literal(fun_name_needle) +1011 p.tag("fun_post", p.negate(p.space() + p.literal("[") + p.until("]") + p.literal("]")) + p.space()) }));1012 });1013 auto fun_res = fun_marker_parser.parse_anywhere_and_extract(clean_haystack);1014 std::string fun_marker = fun_name_needle;1015 if (fun_res.result.success()) {1016 fun_marker = fun_res.tags["fun_marker"];1017 }1018 // now, consume up to two markers, then treat everything up to the function marker as function name prefix1019 auto per_tool_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1020 return p.tag("sec_start", p.marker() + p.space()) + p.tag("call_start", p.marker() + p.space()) +1021 p.tag("fun_pre", p.until(fun_marker)) + fun_marker + p.tag("rest", p.rest());1022 });1023 auto section_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1024 return p.tag("sec_start", p.marker() + p.space()) + fun_marker + p.tag("rest", p.rest());1025 });1026 auto result = per_tool_parser.parse_anywhere_and_extract(clean_haystack);1027 tagged_parse_result result_end;1028 if (result.result.success()) {1029 auto double_closer_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1030 return p.tag("call_end", p.marker() + p.space()) + p.tag("sec_end", p.marker() + p.space()) + p.end();1031 });1032 result_end = double_closer_parser.parse_anywhere_and_extract(result.tags["rest"]);1033 function.name_prefix = fun_res.tags["fun_pre"] + function.name_prefix;1034 } else {1035 result = section_parser.parse_anywhere_and_extract(clean_haystack);1036 auto single_closer_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1037 return p.tag("sec_end", p.marker() + p.space()) + p.end();1038 });1039 result_end = single_closer_parser.parse_anywhere_and_extract(result.tags["rest"]);1040 }1041 format.per_call_start = result.tags["call_start"];1042 format.per_call_end = result_end.tags["call_end"];1043 format.section_start = result.tags["sec_start"];1044 format.section_end = result_end.tags["sec_end"];1045}1046 1047void analyze_tools::check_per_call_markers() {1048 json assistant_one_tool = json{1049 { "role", "assistant" },1050 { "content", "" },1051 { "tool_calls", json::array({ first_tool_call }) }1052 };1053 1054 json assistant_two_tools = json{1055 { "role", "assistant" },1056 { "content", "" },1057 { "tool_calls", json::array({ first_tool_call, second_tool_call }) }1058 };1059 1060 template_params params;1061 params.messages = json::array({ user_msg, assistant_one_tool });1062 params.tools = tools;1063 params.add_generation_prompt = false;1064 params.enable_thinking = true;1065 1066 auto one_vs_two = compare_variants(1067 *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_two_tools }); });1068 1069 if (!one_vs_two) {1070 LOG_DBG(ANSI_ORANGE "%s: Generating double tool call comparison failed\n" ANSI_RESET, __func__);1071 return;1072 }1073 1074 diff_split filter_common_call_part = calculate_diff_split(one_vs_two->diff.suffix, one_vs_two->diff.right);1075 1076 std::string second_tool_content = trim_leading_whitespace(filter_common_call_part.right);1077 if (!format.section_start.empty() &&1078 second_tool_content.find(format.section_start) == 0) {1079 format.per_call_start = format.section_start;1080 format.per_call_end = format.section_end;1081 format.section_start.clear();1082 format.section_end.clear();1083 }1084 1085 if (!format.per_call_end.empty()) {1086 auto count_occurrences = [](const std::string & haystack, const std::string & needle) {1087 size_t count = 0;1088 for (size_t pos = haystack.find(needle); pos != std::string::npos;1089 pos = haystack.find(needle, pos + needle.size())) {1090 count++;1091 }1092 return count;1093 };1094 size_t calls_one = count_occurrences(one_vs_two->output_A, format.per_call_end);1095 size_t calls_two = count_occurrences(one_vs_two->output_B, format.per_call_end);1096 if (calls_one > 0 && calls_one == calls_two) {1097 format.section_end = format.per_call_end;1098 format.per_call_end.clear();1099 }1100 }1101}1102 1103void analyze_tools::extract_function_markers() {1104 json assistant_nocall = json{1105 { "role", "assistant" },1106 { "content", ASSISTANT_MSG },1107 };1108 1109 json assistant_foofoo = json{1110 { "role", "assistant" },1111 { "content", "" },1112 { "tool_calls", json::array({ first_tool_call }) }1113 };1114 1115 json assistant_barbar = json{1116 { "role", "assistant" },1117 { "content", "" },1118 { "tool_calls", json::array({ second_tool_call }) }1119 };1120 1121 template_params params;1122 params.messages = json::array({ user_msg, assistant_foofoo });1123 params.tools = tools;1124 params.add_generation_prompt = false;1125 params.enable_thinking = true;1126 1127 auto comparison = compare_variants(1128 *tmpl, params, [&](template_params & p) { p.messages = json::array({ user_msg, assistant_barbar }); });1129 1130 if (!comparison) {1131 LOG_DBG(ANSI_ORANGE "%s: Template application failed\n" ANSI_RESET, __func__);1132 return;1133 }1134 1135 const auto & diff = comparison->diff;1136 1137 if (diff.left.find(FUN_FIRST) != std::string::npos && diff.right.find(FUN_SECOND) != std::string::npos) {1138 std::string prefix_marker;1139 if (!format.per_call_start.empty()) {1140 prefix_marker = format.per_call_start;1141 } else {1142 prefix_marker = format.section_start;1143 }1144 if (!prefix_marker.empty() && diff.prefix.rfind(prefix_marker) != std::string::npos) {1145 function.name_prefix =1146 diff.prefix.substr(diff.prefix.rfind(prefix_marker) + prefix_marker.size());1147 }1148 1149 // Extract name prefix/suffix from diff.left (stop at the next marker boundary)1150 auto name_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1151 return p.tag("pre", p.until(FUN_FIRST)) + p.literal(FUN_FIRST) +1152 p.tag("post", p.zero_or_more(p.negate(p.marker()) + p.any()));1153 });1154 auto name_result = name_parser.parse_and_extract(diff.left);1155 if (name_result.result.success()) {1156 function.name_prefix += name_result.tags["pre"];1157 function.name_suffix = name_result.tags["post"];1158 }1159 1160 // Extend name_suffix with content from diff.suffix before args begin1161 if (format.mode == tool_format::TAG_WITH_JSON) {1162 // For JSON: name_suffix extends to the first non-marker { or [, including any1163 // markers along the way. Only applies if there's at least one marker after1164 // the JSON content (matching the original "stop < seg_suf.size() - 1" guard).1165 auto suffix_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1166 auto non_json = p.marker() | (p.negate(p.literal("{")) + p.negate(p.literal("[")) + p.any());1167 auto after_json = p.zero_or_more(p.negate(p.marker()) + p.any()) + p.marker();1168 return p.tag("ext", p.zero_or_more(non_json)) + after_json;1169 });1170 auto suf_result = suffix_parser.parse_and_extract(diff.suffix);1171 if (suf_result.result.success()) {1172 function.name_suffix += suf_result.tags["ext"];1173 }1174 } else {1175 // For tagged: name_suffix extends to the first marker (arg marker)1176 auto suffix_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1177 return p.tag("ext", p.zero_or_more(p.negate(p.marker()) + p.any()));1178 });1179 auto suf_result = suffix_parser.parse_and_extract(diff.suffix);1180 if (suf_result.result.success()) {1181 function.name_suffix += suf_result.tags["ext"];1182 1183 auto arg_start = [&](common_peg_parser_builder &p) {1184 return p.marker() + p.space() + p.choice({ p.literal(ARG_FIRST), p.literal(ARG_SECOND) });1185 };1186 auto sep_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {1187 return p.tag("sep", p.zero_or_more(p.negate(arg_start(p)) + p.any())) + arg_start(p);1188 });1189 auto sep_result = sep_parser.parse_and_extract(diff.suffix.substr(suf_result.tags["ext"].size()));1190 if (sep_result.result.success()) {1191 function.args_separator = trim_whitespace(sep_result.tags["sep"]);1192 }1193 }1194 }1195 1196 // Extract the closer (between last arg and call/section end marker)1197 std::string suffix_marker;1198 if (!format.per_call_end.empty()) {1199 suffix_marker = format.per_call_end;1200 } else {