Felipe97/llama-cpp-compiled
01.1k
1// Chat support (incl. tool call grammar constraining & output parsing) w/ generic & custom template handlers.2 3#pragma once4 5#include "common.h"6#include "peg-parser.h"7#include "jinja/parser.h"8#include "jinja/runtime.h"9#include "jinja/caps.h"10 11#include "json.h"12 13#include <chrono>14#include <functional>15#include <map>16#include <string>17#include <vector>18 19using chat_template_caps = jinja::caps;20 21struct common_chat_templates;22 23namespace autoparser {24struct generation_params;25} // namespace autoparser26 27struct common_chat_tool_call {28 std::string name;29 std::string arguments;30 std::string id;31 32 bool operator==(const common_chat_tool_call & other) const {33 return name == other.name && arguments == other.arguments && id == other.id;34 }35};36 37struct common_chat_msg_content_part {38 std::string type;39 std::string text;40 41 // TODO @ngxson : no known chat templates support reasoning_content in content parts yet42 // this can be useful for models with interleaved thinking (like Kimi-K2)43 // if you see any templates explicitly support this, please ping me44 // std::string reasoning_content;45 46 bool operator==(const common_chat_msg_content_part & other) const {47 return type == other.type && text == other.text;48 }49};50 51struct common_chat_template {52 jinja::program prog;53 std::string bos_tok;54 std::string eos_tok;55 std::string src;56 chat_template_caps caps;57 58 common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {59 jinja::lexer lexer;60 auto lexer_res = lexer.tokenize(src);61 this->prog = jinja::parse_from_tokens(lexer_res);62 63 this->src = lexer_res.source;64 this->bos_tok = bos_token;65 this->eos_tok = eos_token;66 67 this->caps = jinja::caps_get(prog);68 // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());69 }70 71 const std::string & source() const { return src; }72 const std::string & bos_token() const { return bos_tok; }73 const std::string & eos_token() const { return eos_tok; }74 75 chat_template_caps original_caps() const {76 return caps;77 }78};79 80struct common_chat_msg {81 std::string role;82 std::string content;83 std::vector<common_chat_msg_content_part> content_parts;84 std::vector<common_chat_tool_call> tool_calls;85 std::string reasoning_content;86 std::string tool_name;87 std::string tool_call_id;88 89 common_json to_json_oaicompat(bool concat_typed_text = false) const;90 91 std::string render_content(const std::string & delimiter = "\n\n") const;92 93 bool empty() const {94 return content.empty() && content_parts.empty() && tool_calls.empty() && reasoning_content.empty() &&95 tool_name.empty() && tool_call_id.empty();96 }97 98 bool contains_media() const {99 for (const auto & part : content_parts) {100 if (part.type == "media_marker") {101 return true;102 }103 }104 return false;105 }106 107 void set_tool_call_ids(std::vector<std::string> & ids_cache,108 const std::function<std::string()> & gen_tool_call_id) {109 for (auto i = 0u; i < tool_calls.size(); i++) {110 if (ids_cache.size() <= i) {111 auto id = tool_calls[i].id;112 if (id.empty()) {113 id = gen_tool_call_id();114 }115 ids_cache.push_back(id);116 }117 tool_calls[i].id = ids_cache[i];118 }119 }120 121 bool operator==(const common_chat_msg & other) const {122 return role == other.role && content == other.content && content_parts == other.content_parts &&123 tool_calls == other.tool_calls && reasoning_content == other.reasoning_content &&124 tool_name == other.tool_name && tool_call_id == other.tool_call_id;125 }126 127 bool operator!=(const common_chat_msg & other) const { return !(*this == other); }128};129 130struct common_chat_msg_diff {131 std::string reasoning_content_delta;132 std::string content_delta;133 size_t tool_call_index = std::string::npos;134 common_chat_tool_call tool_call_delta;135 136 static std::vector<common_chat_msg_diff> compute_diffs(const common_chat_msg & msg_prv,137 const common_chat_msg & msg_new);138 139 bool operator==(const common_chat_msg_diff & other) const {140 return content_delta == other.content_delta && tool_call_index == other.tool_call_index &&141 tool_call_delta == other.tool_call_delta;142 }143};144 145enum common_chat_role {146 COMMON_CHAT_ROLE_UNKNOWN,147 COMMON_CHAT_ROLE_SYSTEM,148 COMMON_CHAT_ROLE_ASSISTANT,149 COMMON_CHAT_ROLE_USER,150 COMMON_CHAT_ROLE_TOOL151};152 153common_chat_role common_chat_role_from_string(const std::string & role);154const char * common_chat_role_to_string(common_chat_role role);155 156struct common_chat_msg_span {157 common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;158 std::size_t pos = 0;159 std::size_t len = 0;160 161 bool valid() const {162 return role != COMMON_CHAT_ROLE_UNKNOWN;163 }164};165 166struct common_chat_msg_spans {167 std::vector<common_chat_msg_span> spans;168 169 void add(common_chat_role role, size_t pos, size_t len) {170 spans.push_back({ role, pos, len });171 }172 173 bool is_user_start(int32_t pos) const {174 for (auto it = spans.begin(); it != spans.end(); ++it) {175 if (it->role == COMMON_CHAT_ROLE_USER && pos == (int32_t) it->pos) {176 return true;177 }178 }179 return false;180 }181 182 int32_t last_user_message_pos() const {183 for (auto it = spans.rbegin(); it != spans.rend(); ++it) {184 if (it->role == COMMON_CHAT_ROLE_USER) {185 return (int32_t) it->pos;186 }187 }188 return -1;189 }190};191 192struct common_chat_msg_delimiter {193 common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN;194 std::string delimiter;195 llama_tokens tokens = {};196};197 198struct common_chat_msg_delimiters {199 std::vector<common_chat_msg_delimiter> delimiters;200 201 common_chat_msg_delimiters() = default;202 common_chat_msg_delimiters(std::initializer_list<common_chat_msg_delimiter> delims) : delimiters(delims) {}203 204 void add(common_chat_role role, const std::string & delimiter) {205 delimiters.push_back({ role, delimiter });206 }207 208 void tokenize(const llama_vocab * vocab);209 210 // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching211 common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;212 213 common_json to_json() const;214};215 216struct common_chat_tool {217 std::string name;218 std::string description;219 std::string parameters;220};221 222enum common_chat_tool_choice {223 COMMON_CHAT_TOOL_CHOICE_AUTO,224 COMMON_CHAT_TOOL_CHOICE_REQUIRED,225 COMMON_CHAT_TOOL_CHOICE_NONE,226};227 228enum common_chat_format {229 COMMON_CHAT_FORMAT_CONTENT_ONLY,230 231 // These are intended to be parsed by the PEG parser232 COMMON_CHAT_FORMAT_PEG_SIMPLE,233 COMMON_CHAT_FORMAT_PEG_NATIVE,234 COMMON_CHAT_FORMAT_PEG_GEMMA4,235 COMMON_CHAT_FORMAT_PEG_MINIMAX_M3,236 237 COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats238};239 240 241// Continuation method provided via `continue_final_message`242enum common_chat_continuation {243 COMMON_CHAT_CONTINUATION_NONE,244 COMMON_CHAT_CONTINUATION_AUTO,245 COMMON_CHAT_CONTINUATION_REASONING,246 COMMON_CHAT_CONTINUATION_CONTENT,247};248 249struct common_chat_templates_inputs {250 std::vector<common_chat_msg> messages;251 std::string grammar;252 std::string json_schema;253 bool add_generation_prompt = true;254 common_chat_continuation continue_final_message = COMMON_CHAT_CONTINUATION_NONE;255 bool use_jinja = true;256 // Parameters below only supported when use_jinja is true257 std::vector<common_chat_tool> tools;258 common_chat_tool_choice tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO;259 bool parallel_tool_calls = false;260 common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool enable_thinking"261 bool enable_thinking = true;262 std::chrono::system_clock::time_point now = std::chrono::system_clock::now();263 std::map<std::string, std::string> chat_template_kwargs;264 bool add_bos = false;265 bool add_eos = false;266 bool force_pure_content = false;267};268 269struct common_chat_params {270 common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;271 std::string prompt;272 std::string grammar;273 bool grammar_lazy = false;274 std::string generation_prompt;275 bool supports_thinking = false;276 std::string thinking_start_tag; // e.g., "<think>"277 std::vector<std::string> thinking_end_tags; // e.g., "</think>"278 std::vector<common_grammar_trigger> grammar_triggers;279 std::vector<std::string> preserved_tokens;280 std::vector<std::string> additional_stops;281 std::string parser;282 common_chat_msg_delimiters message_delimiters;283};284 285// per-message parsing syntax286// should be derived from common_chat_params287struct common_chat_parser_params {288 common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;289 common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"290 // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)291 bool reasoning_in_content = false;292 std::string generation_prompt;293 bool parse_tool_calls = true;294 bool is_continuation = false;295 bool echo = false; // Include assistant prefilled msg in output296 bool debug = false; // Enable debug output for PEG parser297 common_peg_arena parser = {};298 common_chat_parser_params() = default;299 common_chat_parser_params(const common_chat_params & chat_params) {300 format = chat_params.format;301 generation_prompt = chat_params.generation_prompt;302 }303};304 305// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid306bool common_chat_verify_template(const std::string & tmpl, bool use_jinja);307 308void common_chat_templates_free(struct common_chat_templates * tmpls);309 310struct common_chat_templates_deleter {311 void operator()(common_chat_templates * tmpls) { common_chat_templates_free(tmpls); }312};313 314typedef std::unique_ptr<struct common_chat_templates, common_chat_templates_deleter> common_chat_templates_ptr;315 316common_chat_templates_ptr common_chat_templates_init(const struct llama_model * model,317 const std::string & chat_template_override,318 const std::string & bos_token_override = "",319 const std::string & eos_token_override = "");320 321bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls);322std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant = "");323 324struct common_chat_params common_chat_templates_apply(const struct common_chat_templates * tmpls,325 const struct common_chat_templates_inputs & inputs);326 327// Format single message, while taking into account the position of that message in chat history328std::string common_chat_format_single(const struct common_chat_templates * tmpls,329 const std::vector<common_chat_msg> & past_msg,330 const common_chat_msg & new_msg,331 bool add_ass,332 bool use_jinja);333 334// Returns an example of formatted chat335std::string common_chat_format_example(const struct common_chat_templates * tmpls,336 bool use_jinja,337 const std::map<std::string, std::string> & chat_template_kwargs);338 339const char * common_chat_format_name(common_chat_format format);340common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);341common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);342 343// used by arg and server344const char * common_reasoning_format_name(common_reasoning_format format);345common_reasoning_format common_reasoning_format_from_name(const std::string & format);346 347common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice);348 349bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);350 351// Parses a JSON array of messages in OpenAI's chat completion API format.352std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const common_json & messages);353 354std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const common_json & tools);355 356common_chat_continuation common_chat_continuation_parse(const common_json & value);357 358// DEPRECATED: only used in tests359common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);360 361common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);362 363// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments.364common_json common_chat_tool_parameters(const common_json & function);365 366// get template caps, useful for reporting to server /props endpoint367std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);368 369std::string common_chat_template_direct_apply(370 const common_chat_template & tmpl,371 const autoparser::generation_params & inputs);372 373std::string common_chat_template_generation_prompt(374 const common_chat_template & tmpl,375 const autoparser::generation_params & inputs);376 377std::optional<common_chat_params> common_chat_try_specialized_template(378 const common_chat_template & tmpl,379 const std::string & src,380 autoparser::generation_params & params);381 382 383// specialized per-task preset384struct common_chat_prompt_preset {385 std::string system;386 std::string user;387};388 389common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);390 391common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);392