Felipe97/llama-cpp-compiled
01.1k
1#include "chat-peg-parser.h"2 3#include "chat-auto-parser.h"4#include "ggml.h"5#include "peg-parser.h"6 7#include <cstdint>8#include <functional>9 10using ordered_json = common_json;11 12static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {13 int count = 0;14 while (!sv.empty() && std::isspace(static_cast<unsigned char>(sv.back()))) {15 if (max != -1 && count >= max) {16 break;17 }18 sv.remove_suffix(1);19 count++;20 }21 return sv;22}23 24static std::string_view trim_leading_space(std::string_view sv, int max = -1) {25 int count = 0;26 while (!sv.empty() && std::isspace(static_cast<unsigned char>(sv.front()))) {27 if (max != -1 && count >= max) {28 break;29 }30 sv.remove_prefix(1);31 count++;32 }33 return sv;34}35 36static std::string_view trim(std::string_view sv) {37 return trim_trailing_space(trim_leading_space(sv, 1));38}39 40// Count the number of unclosed '{' braces in a JSON-like string,41// properly skipping braces inside quoted strings.42static int json_brace_depth(const std::string & s) {43 int depth = 0;44 bool in_string = false;45 bool escaped = false;46 for (char c : s) {47 if (escaped) {48 escaped = false;49 continue;50 }51 if (c == '\\' && in_string) {52 escaped = true;53 continue;54 }55 if (c == '"') {56 in_string = !in_string;57 continue;58 }59 if (!in_string) {60 if (c == '{') {61 depth++;62 } else if (c == '}') {63 depth--;64 }65 }66 }67 return depth;68}69 70// JSON-escape a string and return the inner content (without surrounding quotes).71static std::string escape_json_string_inner(const std::string & s) {72 std::string escaped = ordered_json(s).dump();73 if (escaped.size() >= 2 && escaped.front() == '"' && escaped.back() == '"') {74 return escaped.substr(1, escaped.size() - 2);75 }76 return escaped;77}78 79// Convert Python-style single-quoted strings to JSON double-quoted strings80// Only converts outer string delimiters, properly handling escape sequences:81// - {'key': 'value'} -> {"key": "value"}82// - {'code': 'print(\'hello\')'} -> {"code": "print('hello')"}83// - {'msg': 'He said "hi"'} -> {"msg": "He said \"hi\""}84static std::string normalize_quotes_to_json(const std::string & input) {85 std::string result;86 result.reserve(input.size() + 16); // May need extra space for escaping87 88 bool in_single_quoted = false;89 bool in_double_quoted = false;90 91 auto is_word_char = [](char ch) { return std::isalnum(static_cast<unsigned char>(ch)) || ch == '_'; };92 93 for (size_t i = 0; i < input.size(); ++i) {94 char c = input[i];95 96 // Handle escape sequences97 if (c == '\\' && i + 1 < input.size()) {98 char next = input[i + 1];99 100 if (in_single_quoted) {101 // Inside a single-quoted string being converted to double quotes102 if (next == '\'') {103 // \' -> ' (escaped single quote becomes unescaped in double-quoted string)104 result += '\'';105 ++i;106 continue;107 }108 if (next == '"') {109 // \" stays as \" (already escaped, works in double-quoted string)110 result += "\\\"";111 ++i;112 continue;113 }114 // Other escapes (\n, \\, etc.): pass through both characters115 result += c;116 result += next;117 ++i;118 continue;119 }120 121 if (in_double_quoted) {122 // Inside a double-quoted string - pass through escape sequences as-is123 result += c;124 result += next;125 ++i;126 continue;127 }128 129 // Outside any string - just pass through the backslash130 result += c;131 continue;132 }133 134 // Handle quote characters135 if (c == '"') {136 if (in_single_quoted) {137 // Unescaped double quote inside single-quoted string -> must escape for JSON138 result += "\\\"";139 } else {140 // Double quote as string delimiter or outside strings141 in_double_quoted = !in_double_quoted;142 result += c;143 }144 } else if (c == '\'') {145 if (in_double_quoted) {146 // Single quote inside double-quoted string -> pass through147 result += c;148 } else if (in_single_quoted) {149 // Closing single quote -> convert to double quote150 in_single_quoted = false;151 result += '"';152 } else {153 // Opening single quote -> convert to double quote154 in_single_quoted = true;155 result += '"';156 }157 } else if (!in_single_quoted && !in_double_quoted && (c == 'T' || c == 'F' || c == 'N') &&158 (i == 0 || !is_word_char(input[i - 1]))) {159 // Python literals -> JSON; prefix match keeps streamed partials monotonic.160 static constexpr std::pair<std::string_view, std::string_view> literals[] = {161 { "True", "true" }, { "False", "false" }, { "None", "null" },162 };163 size_t n = 0;164 while (i + n < input.size() && is_word_char(input[i + n])) {165 ++n;166 }167 std::string_view token(input.data() + i, n);168 bool matched = false;169 for (const auto & [py, js] : literals) {170 if (py.substr(0, n) == token) {171 result += js.substr(0, n);172 i += n - 1;173 matched = true;174 break;175 }176 }177 if (!matched) {178 result += c;179 }180 } else {181 result += c;182 }183 }184 185 return result;186}187 188void tag_based_peg_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) {189 arena.visit(result, [this](const common_peg_ast_node & node) {190 if (!node.tag.empty()) {191 tags[node.tag] = std::string(node.text);192 }193 });194}195 196tagged_parse_result tagged_peg_parser::parse_and_extract(const std::string & input, common_peg_parse_flags extra_flags) const {197 common_peg_parse_context ctx(input, flags | extra_flags);198 auto parse_result = arena.parse(ctx);199 200 tag_based_peg_mapper mapper;201 mapper.from_ast(ctx.ast, parse_result);202 203 return { std::move(parse_result), std::move(mapper.tags) };204}205 206tagged_parse_result tagged_peg_parser::parse_anywhere_and_extract(const std::string & input) const {207 if (input.empty()) {208 return parse_and_extract(input);209 }210 for (size_t i = 0; i < input.size(); i++) {211 common_peg_parse_context ctx(input, flags);212 auto parse_result = arena.parse(ctx, i);213 if (parse_result.success() || i == input.size() - 1) {214 tag_based_peg_mapper mapper;215 mapper.from_ast(ctx.ast, parse_result);216 return { std::move(parse_result), std::move(mapper.tags) };217 }218 }219 GGML_ABORT("Should not happen");220}221 222tagged_peg_parser build_tagged_peg_parser(223 const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {224 common_peg_parser_builder builder;225 builder.set_root(fn(builder));226 return { builder.build() };227}228 229common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::string & tag_name,230 const std::string & marker,231 const common_peg_parser & p) {232 if (marker.empty()) {233 return zero_or_more(choice({ p, rule(tag_name, content(any())) }));234 }235 auto content_chunk = rule(tag_name, content(negate(literal(marker)) + any() + until(marker)));236 return zero_or_more(choice({ p, content_chunk }));237}238 239common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix,240 const std::vector<common_peg_parser> & parsers) {241 if (parsers.empty()) {242 return eps();243 }244 245 if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) {246 return sequence(parsers);247 }248 249 std::map<uint32_t, common_peg_parser> rules;250 std::function<common_peg_parser(uint32_t)> remaining_of;251 252 remaining_of = [&](uint32_t remaining) -> common_peg_parser {253 if (remaining == 0) {254 return eps();255 }256 257 auto cached = rules.find(remaining);258 if (cached != rules.end()) {259 return cached->second;260 }261 262 auto alternatives = choice();263 for (size_t i = 0; i < parsers.size(); i++) {264 const uint32_t bit = 1u << i;265 if (remaining & bit) {266 alternatives |= parsers[i] + remaining_of(remaining & ~bit);267 }268 }269 270 return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second;271 };272 273 return remaining_of((1u << parsers.size()) - 1);274}275 276std::string & common_chat_peg_mapper::args_target() {277 return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer;278}279 280std::string common_chat_peg_mapper::normalize_container_value(const std::string & input) {281 return normalize_quotes_to_json(input);282}283 284void common_chat_peg_mapper::from_ast(const common_peg_ast_arena & arena,285 const common_peg_parse_result & parse_result_arg) {286 arena.visit(parse_result_arg, [this](const common_peg_ast_node & node) { map(node); });287 // Flush any pending tool call that was started but never got a name288 // This happens during partial parsing when the tool call is incomplete289 if (pending_tool_call.has_value() && !pending_tool_call->name.empty()) {290 if (!args_buffer.empty()) {291 pending_tool_call->arguments = args_buffer;292 }293 if (closing_quote_pending && !pending_tool_call->arguments.empty()) {294 pending_tool_call->arguments += "\"";295 }296 result.tool_calls.push_back(pending_tool_call.value());297 pending_tool_call.reset();298 }299 300 // Discard whitespace-only reasoning content (e.g. from <think></think> prefill)301 if (!result.reasoning_content.empty()) {302 bool all_whitespace = true;303 for (char c : result.reasoning_content) {304 if (c != ' ' && c != '\n' && c != '\r' && c != '\t') {305 all_whitespace = false;306 break;307 }308 }309 if (all_whitespace) {310 result.reasoning_content.clear();311 }312 }313}314 315void common_chat_peg_mapper::map(const common_peg_ast_node & node) {316 // Handle reasoning/content tags317 bool is_reasoning = node.tag == common_chat_peg_builder::REASONING;318 bool is_content = node.tag == common_chat_peg_builder::CONTENT;319 320 if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here321 result.reasoning_content += std::string(node.text);322 }323 324 if (is_content) {325 // Concatenate content from multiple content nodes (e.g., when reasoning markers326 // are preserved before content markers in reasoning_format=NONE mode)327 result.content += std::string(node.text);328 }329 330 // Handle tool-related tags (supporting both JSON and tagged formats)331 bool is_tool_open = node.tag == common_chat_peg_builder::TOOL_OPEN;332 bool is_tool_close = node.tag == common_chat_peg_builder::TOOL_CLOSE;333 bool is_tool_name = node.tag == common_chat_peg_builder::TOOL_NAME;334 bool is_tool_id = node.tag == common_chat_peg_builder::TOOL_ID;335 bool is_tool_args = node.tag == common_chat_peg_builder::TOOL_ARGS;336 bool is_arg_open = node.tag == common_chat_peg_builder::TOOL_ARG_OPEN;337 bool is_arg_close = node.tag == common_chat_peg_builder::TOOL_ARG_CLOSE;338 bool is_arg_name = node.tag == common_chat_peg_builder::TOOL_ARG_NAME;339 bool is_arg_value = node.tag == common_chat_peg_builder::TOOL_ARG_VALUE;340 bool is_arg_string_value = node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE;341 342 if (is_tool_open) {343 pending_tool_call = common_chat_tool_call();344 current_tool = &pending_tool_call.value();345 arg_count = 0;346 args_buffer.clear();347 closing_quote_pending = false;348 }349 350 if (is_tool_id && current_tool) {351 auto text = trim_trailing_space(node.text);352 if (text.size() >= 2 && text.front() == '"' && text.back() == '"') {353 text = text.substr(1, text.size() - 2);354 }355 current_tool->id = std::string(text);356 }357 358 if (is_tool_name && current_tool) {359 current_tool->name = std::string(trim_trailing_space(node.text));360 // Now that we have the name, populate the arguments from the buffer361 if (!args_buffer.empty()) {362 current_tool->arguments = args_buffer;363 args_buffer.clear();364 } else if (current_tool->arguments.empty()) {365 current_tool->arguments = "{";366 }367 // Add the tool call to results so streaming can see it368 if (pending_tool_call.has_value()) {369 result.tool_calls.push_back(pending_tool_call.value());370 pending_tool_call.reset();371 current_tool = &result.tool_calls.back();372 }373 }374 375 if (is_tool_args && current_tool) {376 // For JSON format: arguments come as a complete JSON object377 // For tagged format: built up from individual arg_name/arg_value nodes378 auto text = trim_trailing_space(node.text);379 if (!text.empty() && text.front() == '{') {380 args_target() = std::string(text);381 }382 }383 384 if (is_arg_open) {385 closing_quote_pending = false;386 }387 388 if (is_arg_name && current_tool) {389 std::string arg_entry;390 if (arg_count > 0) {391 arg_entry = ",";392 }393 arg_entry += ordered_json(trim(node.text)).dump() + ":";394 ++arg_count;395 396 auto & target = args_target();397 if (target.empty()) {398 target = "{";399 }400 target += arg_entry;401 }402 403 if ((is_arg_value || is_arg_string_value) && current_tool) {404 std::string value_content = std::string(node.text);405 406 std::string value_to_add;407 if (value_content.empty() && is_arg_string_value) {408 // Empty string value - arg_close will add the closing quote409 value_to_add = "\"";410 closing_quote_pending = true;411 } else if (!value_content.empty() && is_arg_string_value) {412 // Schema declares this as string type - always treat as literal string value413 if (!closing_quote_pending) {414 value_to_add = "\"";415 closing_quote_pending = true;416 }417 value_to_add += escape_json_string_inner(value_content);418 } else if (!value_content.empty()) {419 // Pythonic scalars/containers -> JSON.420 value_to_add += normalize_container_value(value_content);421 }422 423 args_target() += value_to_add;424 }425 426 if (is_arg_close && current_tool) {427 if (closing_quote_pending) {428 args_target() += "\"";429 closing_quote_pending = false;430 }431 }432 433 if (is_tool_close && current_tool) {434 // Flush buffer to arguments if tool name was never seen435 if (current_tool->name.empty() && !args_buffer.empty()) {436 current_tool->arguments = args_buffer;437 args_buffer.clear();438 }439 // Close any pending string quote440 if (closing_quote_pending) {441 current_tool->arguments += "\"";442 closing_quote_pending = false;443 }444 // Close any unclosed braces (accounts for nested objects)445 for (int d = json_brace_depth(current_tool->arguments); d > 0; d--) {446 current_tool->arguments += "}";447 }448 // Add tool call to results if named; otherwise discard449 if (pending_tool_call.has_value()) {450 if (!current_tool->name.empty()) {451 result.tool_calls.push_back(pending_tool_call.value());452 }453 pending_tool_call.reset();454 }455 }456}457 458common_peg_parser common_chat_peg_builder::standard_constructed_tools(459 const std::map<std::string, std::string> & markers,460 const ordered_json & tools,461 bool parallel_tool_calls,462 bool force_tool_calls) {463 if (!tools.is_array() || tools.empty()) {464 return eps();465 }466 467 // Extract markers with defaults468 auto get_marker = [&markers](const std::string & key, const std::string & default_val = "") -> std::string {469 auto it = markers.find(key);470 return it != markers.end() ? it->second : default_val;471 };472 473 std::string section_start = get_marker("tool_call_start_marker", "<tool_call>");474 std::string section_end = get_marker("tool_call_end_marker", "</tool_call>");475 std::string func_opener = get_marker("function_opener", "<function=");476 std::string func_name_suffix = get_marker("function_name_suffix", ">");477 std::string func_closer = get_marker("function_closer", "</function>");478 std::string param_key_prefix = get_marker("parameter_key_prefix", "<param=");479 std::string param_key_suffix = get_marker("parameter_key_suffix", ">");480 std::string param_closer = get_marker("parameter_closer", "</param>");481 482 // Build tool choices for tagged format483 auto tool_choices = choice();484 485 for (const auto & tool_def : tools) {486 if (!tool_def.contains("function")) {487 continue;488 }489 const auto & function = tool_def.at("function");490 std::string name = function.at("name");491 ordered_json params = common_chat_tool_parameters(function);492 493 // Build argument parsers494 auto args = eps();495 if (params.contains("properties") && !params["properties"].empty()) {496 auto arg_choice = choice();497 for (const auto & el : params["properties"].items()) {498 const std::string & prop_name = el.key();499 500 auto arg_name_parser =501 choice({ literal(prop_name), literal("\"" + prop_name + "\""), literal("'" + prop_name + "'") });502 503 auto arg_rule = tool_arg(tool_arg_open(literal(param_key_prefix)) + tool_arg_name(arg_name_parser) +504 literal(param_key_suffix) + tool_arg_value(until(param_closer)) +505 tool_arg_close(literal(param_closer)));506 arg_choice |= arg_rule;507 }508 args = zero_or_more(arg_choice + space());509 }510 511 // Build function parser: <function=name>args</function>512 auto tool_parser = tool(tool_open(literal(func_opener) + tool_name(literal(name)) + literal(func_name_suffix)) +513 space() + tool_args(args) + space() + tool_close(literal(func_closer)));514 515 tool_choices |= rule("tool-" + name, tool_parser);516 }517 518 // Build the section with markers519 auto section =520 parallel_tool_calls ?521 trigger_rule("tool-call", literal(section_start) + space() + one_or_more(tool_choices + space()) +522 literal(section_end)) :523 trigger_rule("tool-call", literal(section_start) + space() + tool_choices + space() + literal(section_end));524 525 return force_tool_calls ? section : optional(section);526}527 528// Like python_value(), but the leaf also accepts JSON-cased true/false/null, used by LFM2/LFM2.5529common_peg_parser common_chat_peg_builder::python_or_json_value() {530 return rule("python-or-json-value", [this]() {531 auto ws = space();532 auto value = python_or_json_value();533 534 auto member = sequence({ python_string(), ws, literal(":"), ws, value });535 auto members = sequence({ member, zero_or_more(sequence({ ws, literal(","), ws, member })) });536 auto dict = rule("python-or-json-dict", [&]() {537 return sequence({ literal("{"), ws, choice({ literal("}"), sequence({ members, ws, literal("}") }) }), ws });538 });539 540 auto elements = sequence({ value, zero_or_more(sequence({ literal(","), ws, value })) });541 auto array = rule("python-or-json-array", [&]() {542 return sequence({ literal("["), ws, choice({ literal("]"), sequence({ elements, ws, literal("]") }) }), ws });543 });544 545 return choice({ dict, array, python_string(), python_number(),546 python_bool(), python_null(), json_bool(), json_null() });547 });548}549 550// Python-style tool calls: name(arg1="value1", arg2=123)551// Used only by LFM2 for now, so we don't merge it into autoparser552common_peg_parser common_chat_peg_builder::python_style_tool_calls(553 const ordered_json & tools,554 bool parallel_tool_calls,555 bool allow_json_literals) {556 if (!tools.is_array() || tools.empty()) {557 return eps();558 }559 560 auto tool_choices = choice();561 562 for (const auto & tool_def : tools) {563 if (!tool_def.contains("function")) {564 continue;565 }566 const auto & function = tool_def.at("function");567 std::string name = function.at("name");568 ordered_json params = common_chat_tool_parameters(function);569 570 auto args = eps();571 if (params.contains("properties") && !params["properties"].empty()) {572 auto arg_choice = choice();573 for (const auto & el : params["properties"].items()) {574 const std::string & prop_name = el.key();575 const auto & prop_def = el.value();576 bool is_string_type = (prop_def.contains("type") && prop_def["type"] == "string");577 578 auto arg_name_parser = literal(prop_name);579 580 common_peg_parser arg_value_parser = eps();581 // Quoted literal as a value: normalize_quotes_to_json preserves escapes.582 auto string_value_parser = tool_arg_value(choice({583 literal("\"") + string_content('"') + literal("\""),584 literal("'") + string_content('\'') + literal("'")585 }));586 587 if (is_string_type) {588 arg_value_parser = string_value_parser;589 } else {590 arg_value_parser = tool_arg_value(allow_json_literals ? python_or_json_value() : python_value());591 }592 593 // Full argument: name="value" or name=value594 auto arg_rule = tool_arg(595 tool_arg_open(tool_arg_name(arg_name_parser) + literal("=")) +596 arg_value_parser +597 tool_arg_close(eps())598 );599 arg_choice |= arg_rule;600 }601 602 args = arg_choice + zero_or_more("," + space() + arg_choice);603 }604 605 auto tool_parser = tool(tool_open(tool_name(literal(name)) + literal("(")) +606 space() + tool_args(args) + space() + tool_close(literal(")"))607 );608 609 tool_choices |= rule("tool-" + name, tool_parser);610 }611 612 if (parallel_tool_calls) {613 return "[" + space() + tool_choices + zero_or_more("," + space() + tool_choices) + space() + "]";614 }615 return "[" + space() + tool_choices + space() + "]";616}617 618// Helper: Parse dot notation key into prefix and field name619static std::pair<std::string, std::string> parse_key_spec(const std::string & key) {620 auto dot_pos = key.find('.');621 if (dot_pos == std::string::npos) {622 return {"", key}; // Top-level field623 }624 return {key.substr(0, dot_pos), key.substr(dot_pos + 1)};625}626 627// Mode 1: function_is_key — parse {"function_name": {...}}628common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(629 const ordered_json & tools,630 const std::string & args_key,631 const std::string & effective_args_key,632 const std::string & call_id_key,633 const std::string & gen_call_id_key) {634 635 auto tool_choices = choice();636 637 for (const auto & tool_def : tools) {638 if (!tool_def.contains("function")) {639 continue;640 }641 const auto & function = tool_def.at("function");642 std::string name = function.at("name");643 ordered_json params = common_chat_tool_parameters(function);644 645 // Build inner object fields646 std::vector<common_peg_parser> inner_fields;647 648 if (!call_id_key.empty()) {649 auto id_parser = atomic(650 literal("\"" + call_id_key + "\"") + space() + literal(":") + space() +651 literal("\"") + tool_id(string_content('"')) + literal("\"")652 );653 inner_fields.push_back(optional(id_parser + space() + optional(literal(",") + space())));654 }655 656 if (!gen_call_id_key.empty()) {657 auto gen_id_parser = atomic(658 literal("\"" + gen_call_id_key + "\"") + space() + literal(":") + space() +659 choice({660 literal("\"") + tool_id(string_content('"')) + literal("\""),661 tool_id(json_number())662 })663 );664 inner_fields.push_back(optional(gen_id_parser + space() + optional(literal(",") + space())));665 }666 667 // Arguments — either wrapped in args_key or parsed directly668 common_peg_parser args_parser = eps();669 if (args_key.empty()) {670 args_parser = tool_args(schema(json(), "tool-" + name + "-schema", params));671 } else {672 args_parser = literal("\"" + effective_args_key + "\"") + space() + literal(":") + space() +673 tool_args(schema(json(), "tool-" + name + "-schema", params));674 }675 inner_fields.push_back(args_parser);676 677 // Build inner object parser678 common_peg_parser inner_object = eps();679 if (args_key.empty() && inner_fields.size() == 1) {680 inner_object = inner_fields[0];681 } else {682 inner_object = literal("{") + space();683 for (size_t i = 0; i < inner_fields.size(); i++) {684 inner_object = inner_object + inner_fields[i];685 if (i < inner_fields.size() - 1) {686 inner_object = inner_object + space();687 }688 }689 inner_object = inner_object + space() + literal("}");690 }691 692 auto tool_parser = tool(693 tool_open(literal("{")) + space() +694 literal("\"") + tool_name(literal(name)) + literal("\"") +695 space() + literal(":") + space() +696 inner_object +697 space() + tool_close(literal("}"))698 );699 700 tool_choices |= rule("tool-" + name, tool_parser);701 }702 703 return tool_choices;704}705 706// Mode 2: Nested keys (dot notation like "function.name")707common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(708 const ordered_json & tools,709 const std::string & effective_name_key,710 const std::string & effective_args_key,711 const std::string & call_id_key,712 const std::string & gen_call_id_key) {713 714 auto tool_choices = choice();715 716 auto name_spec = parse_key_spec(effective_name_key);717 auto args_spec = parse_key_spec(effective_args_key);718 719 std::string nested_prefix = !name_spec.first.empty() ? name_spec.first : args_spec.first;720 std::string nested_name_field = !name_spec.first.empty() ? name_spec.second : effective_name_key;721 std::string nested_args_field = !args_spec.first.empty() ? args_spec.second : effective_args_key;722 723 for (const auto & tool_def : tools) {724 if (!tool_def.contains("function")) {725 continue;726 }727 const auto & function = tool_def.at("function");728 std::string name = function.at("name");729 ordered_json params = common_chat_tool_parameters(function);730 731 auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +732 atomic(literal("\"") + tool_name(literal(name)) + literal("\""));733 auto nested_args = literal("\"" + nested_args_field + "\"") + space() + literal(":") + space() +734 tool_args(schema(json(), "tool-" + name + "-schema", params));735 736 auto nested_object = literal("{") + space() +737 nested_name + space() + literal(",") + space() +738 nested_args +739 space() + literal("}");740 741 // Format: { id?, "function": {...} }742 auto tool_parser_body = tool_open(literal("{")) + space();743 744 if (!call_id_key.empty()) {745 auto id_spec = parse_key_spec(call_id_key);746 if (id_spec.first.empty()) {747 auto id_parser = atomic(748 literal("\"" + call_id_key + "\"") + space() + literal(":") + space() +749 literal("\"") + tool_id(string_content('"')) + literal("\"")750 );751 tool_parser_body = tool_parser_body + optional(id_parser + space() + literal(",") + space());752 }753 }754 755 if (!gen_call_id_key.empty()) {756 auto gen_id_spec = parse_key_spec(gen_call_id_key);757 if (gen_id_spec.first.empty()) {758 auto gen_id_parser = atomic(759 literal("\"" + gen_call_id_key + "\"") + space() + literal(":") + space() +760 choice({761 literal("\"") + tool_id(string_content('"')) + literal("\""),762 tool_id(json_number())763 })764 );765 tool_parser_body = tool_parser_body + optional(gen_id_parser + space() + literal(",") + space());766 }767 }768 769 auto nested_field = literal("\"" + nested_prefix + "\"") + space() + literal(":") + space() + nested_object;770 tool_parser_body = tool_parser_body + nested_field + space() + tool_close(literal("}"));771 772 tool_choices |= rule("tool-" + name, tool(tool_parser_body));773 }774 775 return tool_choices;776}777 778// Mode 3: Flat keys with optional ID fields and parameter ordering779common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(780 const ordered_json & tools,781 const std::string & effective_name_key,782 const std::string & effective_args_key,783 const std::string & call_id_key,784 const std::string & gen_call_id_key,785 const std::vector<std::string> & parameters_order,786 bool accept_openai_wrapper) {787 788 auto tool_choices = choice();789 auto name_key_parser = literal("\"" + effective_name_key + "\"");790 auto args_key_parser = literal("\"" + effective_args_key + "\"");791 792 for (const auto & tool_def : tools) {793 if (!tool_def.contains("function")) {794 continue;795 }796 const auto & function = tool_def.at("function");797 std::string name = function.at("name");798 ordered_json params = common_chat_tool_parameters(function);799 800 auto tool_name_ = name_key_parser + space() + literal(":") + space() +801 atomic(literal("\"") + tool_name(literal(name)) + literal("\""));802 auto tool_args_ = args_key_parser + space() + literal(":") + space() +803 tool_args(schema(json(), "tool-" + name + "-schema", params));804 805 // Build ID parsers if keys are provided806 common_peg_parser id_parser = eps();807 if (!call_id_key.empty()) {808 id_parser = atomic(809 literal("\"" + call_id_key + "\"") + space() + literal(":") + space() +810 choice({811 literal("\"") + tool_id(string_content('"')) + literal("\""),812 tool_id(json_number())813 })814 );815 }816 817 common_peg_parser gen_id_parser = eps();818 if (!gen_call_id_key.empty()) {819 gen_id_parser = atomic(820 literal("\"" + gen_call_id_key + "\"") + space() + literal(":") + space() +821 choice({822 literal("\"") + tool_id(string_content('"')) + literal("\""),823 tool_id(json_number())824 })825 );826 }827 828 // Create (parser, key) pairs for all fields, then sort by parameters_order829 std::vector<std::pair<common_peg_parser, std::string>> parser_pairs;830 parser_pairs.emplace_back(tool_name_, effective_name_key);831 parser_pairs.emplace_back(tool_args_, effective_args_key);832 if (!call_id_key.empty()) {833 parser_pairs.emplace_back(optional(id_parser), call_id_key);834 }835 if (!gen_call_id_key.empty()) {836 parser_pairs.emplace_back(optional(gen_id_parser), gen_call_id_key);837 }838 839 std::sort(parser_pairs.begin(), parser_pairs.end(),840 [¶meters_order](const auto & a, const auto & b) {841 auto pos_a = std::find(parameters_order.begin(), parameters_order.end(), a.second);842 auto pos_b = std::find(parameters_order.begin(), parameters_order.end(), b.second);843 size_t idx_a = (pos_a == parameters_order.end()) ? parameters_order.size() : std::distance(parameters_order.begin(), pos_a);844 size_t idx_b = (pos_b == parameters_order.end()) ? parameters_order.size() : std::distance(parameters_order.begin(), pos_b);845 return idx_a < idx_b;846 });847 848 // accept an optional leading "type": "function" field when the model emits the OpenAI wrapper849 common_peg_parser type_field = eps();850 if (accept_openai_wrapper) {851 type_field = optional(literal("\"type\"") + space() + literal(":") + space() +852 literal("\"function\"") + space() + literal(",") + space());853 }854 auto ordered_body = tool_open(literal("{")) + space() + type_field;855 for (size_t i = 0; i < parser_pairs.size(); i++) {856 ordered_body = ordered_body + parser_pairs[i].first;857 if (i < parser_pairs.size() - 1) {858 ordered_body = ordered_body + space() + literal(",") + space();859 }860 }861 ordered_body = ordered_body + space() + tool_close(literal("}"));862 863 tool_choices |= rule("tool-" + name, tool(ordered_body));864 }865 866 return tool_choices;867}868 869common_peg_parser common_chat_peg_builder::prefix(const std::string & s, const std::string & delimiter) {870 if (s.empty()) {871 return eps();872 }873 if (delimiter.empty()) {874 return literal(s);875 }876 return literal(s.substr(0, s.find(delimiter)));877}878 879common_peg_parser common_chat_peg_builder::optspace(const std::string & tag) {880 auto parser = eps();881 size_t end_of_prefix_space = tag.size();882 size_t start_of_suffix_space = tag.size();883 for (size_t i = 0; i < tag.size(); i++) {884 if (!std::isspace(tag[i])) {885 end_of_prefix_space = i;886 break;887 }888 }889 for (size_t i = tag.size(); i > 0; i--) {890 if (!std::isspace(tag[i - 1])) {891 start_of_suffix_space = i;892 break;893 }894 }895 for (size_t i = 0; i < end_of_prefix_space; i++) {896 parser += optional(literal(std::string(1, tag[i])));897 }898 parser += literal(tag.substr(end_of_prefix_space, start_of_suffix_space - end_of_prefix_space));899 for (size_t i = start_of_suffix_space; i < tag.size(); i++) {900 parser += optional(literal(std::string(1, tag[i])));901 }902 return parser;903}904 905common_peg_parser common_chat_peg_builder::standard_json_tools(906 const std::string & section_start,907 const std::string & section_end,908 const ordered_json & tools,909 bool parallel_tool_calls,910 bool force_tool_calls,911 const std::string & name_key,912 const std::string & args_key,913 bool array_wrapped,914 bool function_is_key,915 const std::string & call_id_key,916 const std::string & gen_call_id_key,917 const std::vector<std::string> & parameters_order,918 bool accept_openai_wrapper) {919 if (!tools.is_array() || tools.empty()) {920 return eps();921 }922 923 std::string effective_name_key = name_key.empty() ? "name" : name_key;924 std::string effective_args_key = args_key.empty() ? "arguments" : args_key;925 926 // Dispatch to the appropriate builder based on the JSON layout mode927 common_peg_parser tool_choices = eps();928 if (function_is_key) {929 tool_choices = build_json_tools_function_is_key(tools, args_key, effective_args_key, call_id_key, gen_call_id_key);930 } else {931 auto name_spec = parse_key_spec(effective_name_key);932 auto args_spec = parse_key_spec(effective_args_key);933 if (!name_spec.first.empty() || !args_spec.first.empty()) {934 tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key);935 } else {936 tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper);937 }938 }939 940 // Build the section with markers941 auto tool_calls = tool_choices;942 if (parallel_tool_calls) {943 tool_calls = tool_calls + zero_or_more(space() + literal(",") + space() + tool_choices);944 }945 946 if (array_wrapped) {947 tool_calls = literal("[") + space() + tool_calls + space() + literal("]");948 }949 950 auto section =951 trigger_rule("tool-call", literal(section_start) + space() + tool_calls + space() + literal(section_end));952 953 return force_tool_calls ? section : optional(section);954}955 956void common_chat_peg_gemma4_mapper::from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result) {957 for (const auto & node : result.nodes) {958 visit(arena, node);959 }960}961 962static std::string gemma4_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id) {963 const auto & node = arena.get(id);964 965 if (node.text.empty()) {966 return "";967 }968 969 if (node.rule == "gemma4-number" || node.rule == "gemma4-bool" || node.rule == "gemma4-null") {970 return std::string(node.text);971 }972 973 if (node.rule == "gemma4-string-content") {974 return escape_json_string_inner(std::string(node.text));975 }976 977 if (node.rule == "gemma4-string") {978 std::string result = "\"";979 if (!node.children.empty()) {980 result += gemma4_to_json(arena, node.children[0]);981 if (!node.is_partial) {982 result += "\"";983 }984 }985 return result;986 }987 988 if (node.rule == "gemma4-array") {989 std::string result = "[";990 991 bool add_comma = false;992 for (auto child_id : node.children) {993 if (add_comma) {994 result += ',';995 }996 add_comma = true;997 result += gemma4_to_json(arena, child_id);998 }999 1000 if (!node.is_partial) {1001 result += ']';1002 }1003 return result;1004 }1005 1006 if (node.rule == "gemma4-dict-key-name") {1007 return std::string(node.text);1008 }1009 1010 if (node.rule == "gemma4-dict-key") {1011 std::string result = "\"";1012 if (!node.children.empty()) {1013 result += escape_json_string_inner(gemma4_to_json(arena, node.children[0]));1014 }1015 if (!node.is_partial) {1016 result += "\":";1017 }1018 return result;1019 }1020 1021 if (node.rule == "gemma4-dict-kv") {1022 std::string result;1023 for (auto child_id : node.children) {1024 result += gemma4_to_json(arena, child_id);1025 }1026 return result;1027 }1028 1029 if (node.rule == "gemma4-dict") {1030 std::string result = "{";1031 1032 bool add_comma = false;1033 for (auto child_id : node.children) {1034 if (add_comma) {1035 result += ',';1036 }1037 add_comma = true;1038 result += gemma4_to_json(arena, child_id);1039 }1040 1041 if (!node.is_partial) {1042 result += '}';1043 }1044 return result;1045 }1046 1047 if (node.rule == "gemma4-value") {1048 if (!node.children.empty()) {1049 return gemma4_to_json(arena, node.children[0]);1050 }1051 return "";1052 }1053 1054 return "";1055}1056 1057void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) {1058 const auto & node = arena.get(id);1059 1060 if (node.tag == "reasoning") {1061 result.reasoning_content += std::string(node.text);1062 return;1063 }1064 1065 if (node.tag == "content") {1066 result.content += std::string(node.text);1067 return;1068 }1069 1070 if (node.tag == "tool") {1071 auto name_id = arena.find_by_tag(node, "tool-name");1072 auto args_id = arena.find_by_tag(node, "tool-args");1073 1074 if (name_id != COMMON_PEG_INVALID_AST_ID && args_id != COMMON_PEG_INVALID_AST_ID) {1075 const auto & name_node = arena.get(name_id);1076 const auto & args_node = arena.get(args_id);1077 1078 if (!name_node.is_partial) {1079 common_chat_tool_call call;1080 call.name = std::string(name_node.text);1081 if (!args_node.children.empty()) {1082 call.arguments = gemma4_to_json(arena, args_node.children[0]);1083 }1084 result.tool_calls.push_back(call);1085 }1086 }1087 1088 return;1089 }1090 1091 for (auto child_id : node.children) {1092 visit(arena, child_id);1093 }1094}1095 1096static void minimax_m3_collect(const common_peg_ast_arena & arena,1097 const common_peg_ast_node & node,1098 const std::string & tag,1099 std::vector<common_peg_ast_id> & out) {1100 for (auto child_id : node.children) {1101 const auto & child = arena.get(child_id);1102 if (child.tag == tag) {1103 out.push_back(child_id);1104 } else {1105 minimax_m3_collect(arena, child, tag, out);1106 }1107 }1108}1109 1110static common_peg_ast_id minimax_m3_value_of(const common_peg_ast_arena & arena, const common_peg_ast_node & node) {1111 for (auto child_id : node.children) {1112 const auto & tag = arena.get(child_id).tag;1113 if (tag == common_chat_peg_builder::TOOL_ARG_VALUE ||1114 tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE ||1115 tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT ||1116 tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) {1117 return child_id;1118 }1119 }1120 return COMMON_PEG_INVALID_AST_ID;1121}1122 1123static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed);1124 1125static std::string minimax_m3_member_to_json(const common_peg_ast_arena & arena, const common_peg_ast_node & node) {1126 auto name_id = arena.find_by_tag(node, common_chat_peg_builder::TOOL_ARG_NAME);1127 if (name_id == COMMON_PEG_INVALID_AST_ID) {1128 return "";1129 }1130 1131 return ordered_json(arena.get(name_id).text).dump() + ":" +1132 minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, node), !node.is_partial);1133}1134 1135static std::string minimax_m3_container_to_json(const common_peg_ast_arena & arena,1136 const common_peg_ast_node & node,1137 bool is_object,1138 bool closed) {1139 const std::string tag = is_object ? common_chat_peg_builder::TOOL_ARG1140 : common_chat_peg_minimax_m3_mapper::TOOL_ARG_ITEM;1141 1142 std::vector<common_peg_ast_id> entries;1143 minimax_m3_collect(arena, node, tag, entries);1144 1145 std::string result = is_object ? "{" : "[";1146 1147 bool add_comma = false;1148 for (auto entry_id : entries) {1149 const auto & entry = arena.get(entry_id);1150 1151 std::string text;1152 if (is_object) {1153 text = minimax_m3_member_to_json(arena, entry);1154 } else {1155 text = minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, entry), !entry.is_partial);1156 }1157 1158 if (text.empty()) {1159 continue;1160 }1161 1162 if (add_comma) {1163 result += ",";1164 }1165 add_comma = true;1166 result += text;1167 }1168 1169 if (closed) {1170 result += is_object ? "}" : "]";1171 }1172 return result;1173}1174 1175static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed) {1176 if (id == COMMON_PEG_INVALID_AST_ID) {1177 return "";1178 }1179 1180 const auto & node = arena.get(id);1181 1182 if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT) {1183 return minimax_m3_container_to_json(arena, node, /* is_object = */ true, closed);1184 }1185 1186 if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) {1187 return minimax_m3_container_to_json(arena, node, /* is_object = */ false, closed);1188 }1189 1190 if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) {1191 return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : "");1192 }1193 1194 // Numbers and booleans are written verbatim by the template1195 return std::string(node.text);1196}1197 1198void common_chat_peg_minimax_m3_mapper::from_ast(const common_peg_ast_arena & arena,1199 const common_peg_parse_result & result) {1200 for (const auto & node : result.nodes) {