Felipe97/llama-cpp-compiled

1#pragma once2 3#include "chat.h"4#include "peg-parser.h"5 6#include <map>7#include <optional>8#include <vector>9 10class common_chat_peg_mapper {11  public:12    common_chat_msg & result;13 14    common_chat_peg_mapper(common_chat_msg & msg) : result(msg) {}15 16    virtual ~common_chat_peg_mapper() = default;17 18    virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);19    virtual void map(const common_peg_ast_node & node);20  protected:21    virtual std::string normalize_container_value(const std::string & input);22  private:23      // Tool call handling state24      std::optional<common_chat_tool_call> pending_tool_call;  // Tool call waiting for name25      common_chat_tool_call *              current_tool          = nullptr;26      int                                  arg_count             = 0;27      bool                                 closing_quote_pending = false;28      std::string                          args_buffer;  // Buffer to delay arguments until tool name is known29 30      // Returns a reference to the active argument destination string.31      // Before tool_name is known, writes go to args_buffer; after, to current_tool->arguments.32      std::string & args_target();33};34 35class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper {36  public:37    common_chat_peg_gemma4_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}38    virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);39  private:40    void visit(const common_peg_ast_arena & arena, common_peg_ast_id id);41};42 43class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper {44  public:45    static constexpr const char * TOOL_ARG_OBJECT = "tool-arg-object";46    static constexpr const char * TOOL_ARG_ARRAY  = "tool-arg-array";47    static constexpr const char * TOOL_ARG_ITEM   = "tool-arg-item";48 49    common_chat_peg_minimax_m3_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {}50    virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);51  private:52    void visit(const common_peg_ast_arena & arena, common_peg_ast_id id);53};54 55struct content_structure;56struct tool_call_structure;57 58constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6;59 60class common_chat_peg_builder : public common_peg_parser_builder {61  public:62    // Tag constants (from former common_chat_peg_base_builder)63    static constexpr const char * REASONING_BLOCK = "reasoning-block";64    static constexpr const char * REASONING       = "reasoning";65    static constexpr const char * CONTENT         = "content";66 67    // Tag constants68    static constexpr const char * TOOL           = "tool";69    static constexpr const char * TOOL_OPEN      = "tool-open";70    static constexpr const char * TOOL_CLOSE     = "tool-close";71    static constexpr const char * TOOL_ID        = "tool-id";72    static constexpr const char * TOOL_NAME      = "tool-name";73    static constexpr const char * TOOL_ARGS      = "tool-args";74    static constexpr const char * TOOL_ARG       = "tool-arg";75    static constexpr const char * TOOL_ARG_OPEN  = "tool-arg-open";76    static constexpr const char * TOOL_ARG_CLOSE = "tool-arg-close";77    static constexpr const char * TOOL_ARG_NAME         = "tool-arg-name";78    static constexpr const char * TOOL_ARG_VALUE        = "tool-arg-value";79    static constexpr const char * TOOL_ARG_STRING_VALUE = "tool-arg-string-value";  // For schema-declared string types80 81    // Low-level tag methods (from former common_chat_peg_base_builder)82    common_peg_parser reasoning_block(const common_peg_parser & p) { return tag(REASONING_BLOCK, p); }83 84    common_peg_parser reasoning(const common_peg_parser & p) { return tag(REASONING, p); }85 86    common_peg_parser content(const common_peg_parser & p) { return tag(CONTENT, p); }87 88    common_peg_parser tag_with_safe_content(const std::string &       tag_name,89                        const std::string &       marker,90                        const common_peg_parser & p);91 92    // Low-level tag methods93    common_peg_parser tool(const common_peg_parser & p) { return tag(TOOL, p); }94    common_peg_parser tool_open(const common_peg_parser & p) { return atomic(tag(TOOL_OPEN, p)); }95    common_peg_parser tool_close(const common_peg_parser & p) { return atomic(tag(TOOL_CLOSE, p)); }96    common_peg_parser tool_id(const common_peg_parser & p) { return atomic(tag(TOOL_ID, p)); }97    common_peg_parser tool_name(const common_peg_parser & p) { return atomic(tag(TOOL_NAME, p)); }98    common_peg_parser tool_args(const common_peg_parser & p) { return tag(TOOL_ARGS, p); }99    common_peg_parser tool_arg(const common_peg_parser & p) { return tag(TOOL_ARG, p); }100    common_peg_parser tool_arg_open(const common_peg_parser & p) { return atomic(tag(TOOL_ARG_OPEN, p)); }101    common_peg_parser tool_arg_close(const common_peg_parser & p) { return atomic(tag(TOOL_ARG_CLOSE, p)); }102    common_peg_parser tool_arg_name(const common_peg_parser & p) { return atomic(tag(TOOL_ARG_NAME, p)); }103    common_peg_parser tool_arg_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); }104 105    // Use for schema-declared string types - won't be treated as potential JSON container106    common_peg_parser tool_arg_string_value(const common_peg_parser & p) { return tag(TOOL_ARG_STRING_VALUE, p); }107    common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); }108 109 110    // Matches every parser exactly once, in any order.111    common_peg_parser permute(const std::string & rule_prefix, const std::vector<common_peg_parser> & parsers);112 113    // Return a parser that parses the prefix of a string, up to a given delimiter.114    common_peg_parser prefix(const std::string & s, const std::string & delimiter = {});115 116    // Return a parser that parses all elements of tag, but leading and trailing spaces are optional117    common_peg_parser optspace(const std::string & tag);118 119    // Legacy-compatible helper for building standard JSON tool calls120    // Used by tests and manual parsers121    // name_key/args_key: JSON key names for function name and arguments122    //   Empty or "name"/"arguments" will accept both common variations123    //   Supports dot notation for nested objects (e.g., "function.name")124    // array_wrapped: if true, tool calls are wrapped in JSON array [...]125    // function_is_key: if true, function name is the JSON key (e.g., {"func_name": {...}})126    // call_id_key: JSON key for string call ID (e.g., "id")127    // gen_call_id_key: JSON key for generated integer call ID (e.g., "tool_call_id")128    // parameters_order: order in which JSON fields should be parsed129    common_peg_parser standard_json_tools(const std::string &              section_start,130                                          const std::string &              section_end,131                                          const common_json &   tools,132                                          bool                             parallel_tool_calls,133                                          bool                             force_tool_calls,134                                          const std::string &              name_key = "",135                                          const std::string &              args_key = "",136                                          bool                             array_wrapped = false,137                                          bool                             function_is_key = false,138                                          const std::string &              call_id_key = "",139                                          const std::string &              gen_call_id_key = "",140                                          const std::vector<std::string> & parameters_order = {},141                                          bool                             accept_openai_wrapper = false);142 143    // Legacy-compatible helper for building XML/tagged style tool calls144    // Used by tests and manual parsers145    common_peg_parser standard_constructed_tools(const std::map<std::string, std::string> & markers,146                                                 const common_json &             tools,147                                                 bool                                       parallel_tool_calls,148                                                 bool                                       force_tool_calls);149 150    // Helper for Python-style function call format: name(arg1="value1", arg2=123)151    // Used by LFM2 and similar templates152    common_peg_parser python_style_tool_calls(const common_json & tools,153                                              bool                           parallel_tool_calls,154                                              bool                           allow_json_literals);155 156  private:157    // Python values plus JSON true/false/null.158    common_peg_parser python_or_json_value();159 160    // Implementation helpers for standard_json_tools — one per JSON tool call layout mode161    common_peg_parser build_json_tools_function_is_key(const common_json & tools,162                                                       const std::string &            args_key,163                                                       const std::string &            effective_args_key,164                                                       const std::string &            call_id_key,165                                                       const std::string &            gen_call_id_key);166 167    common_peg_parser build_json_tools_nested_keys(const common_json & tools,168                                                   const std::string &            effective_name_key,169                                                   const std::string &            effective_args_key,170                                                   const std::string &            call_id_key,171                                                   const std::string &            gen_call_id_key);172 173    common_peg_parser build_json_tools_flat_keys(const common_json &   tools,174                                                 const std::string &              effective_name_key,175                                                 const std::string &              effective_args_key,176                                                 const std::string &              call_id_key,177                                                 const std::string &              gen_call_id_key,178                                                 const std::vector<std::string> & parameters_order,179                                                 bool                             accept_openai_wrapper);180};181 182inline common_peg_arena build_chat_peg_parser(183  const std::function<common_peg_parser(common_chat_peg_builder & builder)> & fn) {184  common_chat_peg_builder builder;185  builder.set_root(fn(builder));186  return builder.build();187}188 189class tag_based_peg_mapper {190  public:191    std::map<std::string, std::string> tags;192 193    void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result);194};195 196struct tagged_parse_result {197    common_peg_parse_result              result;198    std::map<std::string, std::string> tags;199};200 201struct tagged_peg_parser {202    common_peg_arena arena;203    common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE;204 205    tagged_peg_parser & withDebug() {206      flags |= COMMON_PEG_PARSE_FLAG_DEBUG;207      return *this;208    }209 210    tagged_peg_parser & withoutDebug() {211      flags = flags & ~COMMON_PEG_PARSE_FLAG_DEBUG;212      return *this;213    }214 215    tagged_parse_result parse_and_extract(const std::string & input, common_peg_parse_flags extra_flags = COMMON_PEG_PARSE_FLAG_NONE) const;216    tagged_parse_result parse_anywhere_and_extract(const std::string & input) const;217};218 219tagged_peg_parser build_tagged_peg_parser(220    const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn);221