Felipe97/llama-cpp-compiled

1#include "llama-chat.h"2 3#include "llama.h"4 5#include <map>6#include <sstream>7#include <algorithm>8 9#if __cplusplus >= 202000L10    #define LU8(x) (const char*)(u8##x)11#else12    #define LU8(x) u8##x13#endif14 15// trim whitespace from the beginning and end of a string16static std::string trim(const std::string & str) {17    size_t start = 0;18    size_t end = str.size();19    while (start < end && isspace(static_cast<unsigned char>(str[start]))) {20        start += 1;21    }22    while (end > start && isspace(static_cast<unsigned char>(str[end - 1]))) {23        end -= 1;24    }25    return str.substr(start, end - start);26}27 28static const std::map<std::string, llm_chat_template> LLM_CHAT_TEMPLATES = {29    { "chatml",            LLM_CHAT_TEMPLATE_CHATML            },30    { "llama2",            LLM_CHAT_TEMPLATE_LLAMA_2           },31    { "llama2-sys",        LLM_CHAT_TEMPLATE_LLAMA_2_SYS       },32    { "llama2-sys-bos",    LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS   },33    { "llama2-sys-strip",  LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP },34    { "mistral-v1",        LLM_CHAT_TEMPLATE_MISTRAL_V1        },35    { "mistral-v3",        LLM_CHAT_TEMPLATE_MISTRAL_V3        },36    { "mistral-v3-tekken", LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN },37    { "mistral-v7",        LLM_CHAT_TEMPLATE_MISTRAL_V7        },38    { "mistral-v7-tekken", LLM_CHAT_TEMPLATE_MISTRAL_V7_TEKKEN },39    { "phi3",              LLM_CHAT_TEMPLATE_PHI_3             },40    { "phi4",              LLM_CHAT_TEMPLATE_PHI_4             },41    { "falcon3",           LLM_CHAT_TEMPLATE_FALCON_3          },42    { "zephyr",            LLM_CHAT_TEMPLATE_ZEPHYR            },43    { "monarch",           LLM_CHAT_TEMPLATE_MONARCH           },44    { "gemma",             LLM_CHAT_TEMPLATE_GEMMA             },45    { "orion",             LLM_CHAT_TEMPLATE_ORION             },46    { "openchat",          LLM_CHAT_TEMPLATE_OPENCHAT          },47    { "vicuna",            LLM_CHAT_TEMPLATE_VICUNA            },48    { "vicuna-orca",       LLM_CHAT_TEMPLATE_VICUNA_ORCA       },49    { "deepseek",          LLM_CHAT_TEMPLATE_DEEPSEEK          },50    { "deepseek2",         LLM_CHAT_TEMPLATE_DEEPSEEK_2        },51    { "deepseek3",         LLM_CHAT_TEMPLATE_DEEPSEEK_3        },52    { "deepseek-ocr",      LLM_CHAT_TEMPLATE_DEEPSEEK_OCR      },53    { "command-r",         LLM_CHAT_TEMPLATE_COMMAND_R         },54    { "llama3",            LLM_CHAT_TEMPLATE_LLAMA_3           },55    { "chatglm3",          LLM_CHAT_TEMPLATE_CHATGLM_3         },56    { "chatglm4",          LLM_CHAT_TEMPLATE_CHATGLM_4         },57    { "glmedge",           LLM_CHAT_TEMPLATE_GLMEDGE           },58    { "minicpm",           LLM_CHAT_TEMPLATE_MINICPM           },59    { "exaone3",           LLM_CHAT_TEMPLATE_EXAONE_3          },60    { "exaone4",           LLM_CHAT_TEMPLATE_EXAONE_4          },61    { "exaone-moe",        LLM_CHAT_TEMPLATE_EXAONE_MOE        },62    { "rwkv-world",        LLM_CHAT_TEMPLATE_RWKV_WORLD        },63    { "granite",           LLM_CHAT_TEMPLATE_GRANITE_3_X       },64    { "granite-4.0",       LLM_CHAT_TEMPLATE_GRANITE_4_0       },65    { "granite-4.1",       LLM_CHAT_TEMPLATE_GRANITE_4_1       },66    { "gigachat",          LLM_CHAT_TEMPLATE_GIGACHAT          },67    { "megrez",            LLM_CHAT_TEMPLATE_MEGREZ            },68    { "yandex",            LLM_CHAT_TEMPLATE_YANDEX            },69    { "bailing",           LLM_CHAT_TEMPLATE_BAILING           },70    { "bailing-think",     LLM_CHAT_TEMPLATE_BAILING_THINK     },71    { "bailing2",          LLM_CHAT_TEMPLATE_BAILING2          },72    { "llama4",            LLM_CHAT_TEMPLATE_LLAMA4            },73    { "smolvlm",           LLM_CHAT_TEMPLATE_SMOLVLM           },74    { "hunyuan-moe",       LLM_CHAT_TEMPLATE_HUNYUAN_MOE       },75    { "gpt-oss",           LLM_CHAT_TEMPLATE_OPENAI_MOE        },76    { "hunyuan-dense",     LLM_CHAT_TEMPLATE_HUNYUAN_DENSE     },77    { "hunyuan-vl",        LLM_CHAT_TEMPLATE_HUNYUAN_VL        },78    { "kimi-k2",           LLM_CHAT_TEMPLATE_KIMI_K2           },79    { "seed_oss",          LLM_CHAT_TEMPLATE_SEED_OSS          },80    { "grok-2",            LLM_CHAT_TEMPLATE_GROK_2            },81    { "pangu-embedded",    LLM_CHAT_TEMPLATE_PANGU_EMBED       },82    { "solar-open",        LLM_CHAT_TEMPLATE_SOLAR_OPEN        },83};84 85llm_chat_template llm_chat_template_from_str(const std::string & name) {86    return LLM_CHAT_TEMPLATES.at(name);87}88 89llm_chat_template llm_chat_detect_template(const std::string & tmpl) {90    try {91        return llm_chat_template_from_str(tmpl);92    } catch (const std::out_of_range &) {93        // ignore94    }95 96    auto tmpl_contains = [&tmpl](const char * haystack) -> bool {97        return tmpl.find(haystack) != std::string::npos;98    };99    if (tmpl_contains("<|im_start|>")) {100        return tmpl_contains("<|im_sep|>")101            ? LLM_CHAT_TEMPLATE_PHI_4102            : tmpl_contains("<end_of_utterance>")103                ? LLM_CHAT_TEMPLATE_SMOLVLM // SmolVLM uses <|im_start|> as BOS, but it is NOT chatml104                : LLM_CHAT_TEMPLATE_CHATML;105    } else if (tmpl.find("mistral") == 0 || tmpl_contains("[INST]")) {106        if (tmpl_contains("[SYSTEM_PROMPT]")) {107            return LLM_CHAT_TEMPLATE_MISTRAL_V7;108        } else if (109            // catches official 'v1' template110            tmpl_contains("' [INST] ' + system_message")111            // catches official 'v3' and 'v3-tekken' templates112            || tmpl_contains("[AVAILABLE_TOOLS]")113        ) {114            // Official mistral 'v1', 'v3' and 'v3-tekken' templates115            // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/chat_templates.md116            // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/templates.md117            if (tmpl_contains(" [INST]")) {118                return LLM_CHAT_TEMPLATE_MISTRAL_V1;119            } else if (tmpl_contains("\"[INST]\"")) {120                return LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN;121            }122            return LLM_CHAT_TEMPLATE_MISTRAL_V3;123        } else {124            // llama2 template and its variants125            // [variant] support system message126            // See: https://huggingface.co/blog/llama2#how-to-prompt-llama-2127            bool support_system_message = tmpl_contains("<<SYS>>");128            bool add_bos_inside_history = tmpl_contains("bos_token + '[INST]");129            bool strip_message = tmpl_contains("content.strip()");130            if (strip_message) {131                return LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP;132            } else if (add_bos_inside_history) {133                return LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS;134            } else if (support_system_message) {135                return LLM_CHAT_TEMPLATE_LLAMA_2_SYS;136            } else {137                return LLM_CHAT_TEMPLATE_LLAMA_2;138            }139        }140    } else if (tmpl_contains("<|assistant|>") && tmpl_contains("<|end|>")) {141        return LLM_CHAT_TEMPLATE_PHI_3;142    } else if (tmpl_contains("[gMASK]<sop>")) {143        return LLM_CHAT_TEMPLATE_CHATGLM_4;144    } else if (tmpl_contains("<|assistant|>") && tmpl_contains("<|user|>")) {145        if (tmpl_contains("<|tool_declare|>")) {146            return LLM_CHAT_TEMPLATE_EXAONE_MOE;147        }148        return tmpl_contains("</s>") ? LLM_CHAT_TEMPLATE_FALCON_3 : LLM_CHAT_TEMPLATE_GLMEDGE;149    } else if (tmpl_contains("<|{{ item['role'] }}|>") && tmpl_contains("<|begin_of_image|>")) {150        return LLM_CHAT_TEMPLATE_GLMEDGE;151    } else if (tmpl_contains("<|user|>") && tmpl_contains("<|endoftext|>")) {152        return LLM_CHAT_TEMPLATE_ZEPHYR;153    } else if (tmpl_contains("bos_token + message['role']")) {154        return LLM_CHAT_TEMPLATE_MONARCH;155    } else if (tmpl_contains("<start_of_turn>")) {156        return LLM_CHAT_TEMPLATE_GEMMA;157    } else if (tmpl_contains("'\\n\\nAssistant: ' + eos_token")) {158        // OrionStarAI/Orion-14B-Chat159        return LLM_CHAT_TEMPLATE_ORION;160    } else if (tmpl_contains("GPT4 Correct ")) {161        // openchat/openchat-3.5-0106162        return LLM_CHAT_TEMPLATE_OPENCHAT;163    } else if (tmpl_contains("USER: ") && tmpl_contains("ASSISTANT: ")) {164        // eachadea/vicuna-13b-1.1 (and Orca variant)165        if (tmpl_contains("SYSTEM: ")) {166            return LLM_CHAT_TEMPLATE_VICUNA_ORCA;167        }168        return LLM_CHAT_TEMPLATE_VICUNA;169    } else if (tmpl_contains("### Instruction:") && tmpl_contains("<|EOT|>")) {170        // deepseek-ai/deepseek-coder-33b-instruct171        return LLM_CHAT_TEMPLATE_DEEPSEEK;172    } else if (tmpl_contains("<|START_OF_TURN_TOKEN|>") && tmpl_contains("<|USER_TOKEN|>")) {173        // CohereForAI/c4ai-command-r-plus174        return LLM_CHAT_TEMPLATE_COMMAND_R;175    } else if (tmpl_contains("<|start_header_id|>") && tmpl_contains("<|end_header_id|>")) {176        return LLM_CHAT_TEMPLATE_LLAMA_3;177    } else if (tmpl_contains("[gMASK]sop")) {178        // chatglm3-6b179        return LLM_CHAT_TEMPLATE_CHATGLM_3;180    } else if (tmpl_contains(LU8("<用户>"))) {181        // MiniCPM-3B-OpenHermes-2.5-v2-GGUF182        return LLM_CHAT_TEMPLATE_MINICPM;183    } else if (tmpl_contains("'Assistant: ' + message['content'] + eos_token")) {184        return LLM_CHAT_TEMPLATE_DEEPSEEK_2;185    } else if (tmpl_contains(LU8("<｜Assistant｜>")) && tmpl_contains(LU8("<｜User｜>")) && tmpl_contains(LU8("<｜end▁of▁sentence｜>"))) {186        return LLM_CHAT_TEMPLATE_DEEPSEEK_3;187    } else if (tmpl_contains("[|system|]") && tmpl_contains("[|assistant|]") && tmpl_contains("[|endofturn|]")) {188        if (tmpl_contains("[|tool|]")) {189            return LLM_CHAT_TEMPLATE_EXAONE_4;190        }191        // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb192        // EXAONE-3.0-7.8B-Instruct193        return LLM_CHAT_TEMPLATE_EXAONE_3;194    } else if (tmpl_contains("rwkv-world") || tmpl_contains("{{- 'User: ' + message['content']|trim + '\\n\\n' -}}")) {195        return LLM_CHAT_TEMPLATE_RWKV_WORLD;196    } else if (tmpl_contains("<|start_of_role|>")) {197        if (tmpl_contains("<tool_call>") || tmpl_contains("<tools>")) {198            if (tmpl_contains("g4_default_system_message")) {199                return LLM_CHAT_TEMPLATE_GRANITE_4_0;200            }201            return LLM_CHAT_TEMPLATE_GRANITE_4_1;202        }203        return LLM_CHAT_TEMPLATE_GRANITE_3_X;204    } else if (tmpl_contains("message['role'] + additional_special_tokens[0] + message['content'] + additional_special_tokens[1]")) {205        return LLM_CHAT_TEMPLATE_GIGACHAT;206    } else if (tmpl_contains("<|role_start|>")) {207        return LLM_CHAT_TEMPLATE_MEGREZ;208    } else if (tmpl_contains(" Ассистент:")) {209        return LLM_CHAT_TEMPLATE_YANDEX;210    } else if (tmpl_contains("<role>ASSISTANT</role>") && tmpl_contains("'HUMAN'")) {211        return LLM_CHAT_TEMPLATE_BAILING;212    } else if (tmpl_contains("<role>ASSISTANT</role>") && tmpl_contains("\"HUMAN\"") && tmpl_contains("<think>")) {213        return LLM_CHAT_TEMPLATE_BAILING_THINK;214    } else if (tmpl_contains("<role>ASSISTANT</role>") && tmpl_contains("<role>HUMAN</role>") && tmpl_contains("<|role_end|>")) {215        return LLM_CHAT_TEMPLATE_BAILING2;216    } else if (tmpl_contains("<|header_start|>") && tmpl_contains("<|header_end|>")) {217        return LLM_CHAT_TEMPLATE_LLAMA4;218    } else if (tmpl_contains("<|endofuserprompt|>")) {219        return LLM_CHAT_TEMPLATE_DOTS1;220    } else if (tmpl_contains("<|extra_0|>") && tmpl_contains("<|extra_4|>")) {221        return LLM_CHAT_TEMPLATE_HUNYUAN_MOE;222    } else if (tmpl_contains("<|start|>") && tmpl_contains("<|channel|>")) {223        return LLM_CHAT_TEMPLATE_OPENAI_MOE;224    } else if (tmpl_contains("<｜hy_Assistant｜>") && tmpl_contains("<｜hy_begin▁of▁sentence｜>")) {225        return LLM_CHAT_TEMPLATE_HUNYUAN_VL;226    } else if (tmpl_contains("<｜hy_Assistant｜>") && tmpl_contains("<｜hy_place▁holder▁no▁3｜>")) {227        return LLM_CHAT_TEMPLATE_HUNYUAN_DENSE;228    } else if (tmpl_contains("<|im_assistant|>assistant<|im_middle|>")) {229        return LLM_CHAT_TEMPLATE_KIMI_K2;230    } else if (tmpl_contains("<seed:bos>")) {231        return LLM_CHAT_TEMPLATE_SEED_OSS;232    } else if (tmpl_contains("'Assistant: '  + message['content'] + '<|separator|>")) {233        return LLM_CHAT_TEMPLATE_GROK_2;234    } else if (tmpl_contains(LU8("[unused9]系统：[unused10]"))) {235        return LLM_CHAT_TEMPLATE_PANGU_EMBED;236    } else if (tmpl_contains("<|begin|>") && tmpl_contains("<|end|>") && tmpl_contains("<|content|>")) {237        return LLM_CHAT_TEMPLATE_SOLAR_OPEN;238    }239    return LLM_CHAT_TEMPLATE_UNKNOWN;240}241 242// Simple version of "llama_apply_chat_template" that only works with strings243// This function uses heuristic checks to determine commonly used template. It is not a jinja parser.244int32_t llm_chat_apply_template(245    llm_chat_template tmpl,246    const std::vector<const llama_chat_message *> & chat,247    std::string & dest, bool add_ass) {248    // Taken from the research: https://github.com/ggml-org/llama.cpp/issues/5527249    std::stringstream ss;250    if (tmpl == LLM_CHAT_TEMPLATE_CHATML) {251        // chatml template252        for (auto message : chat) {253            ss << "<|im_start|>" << message->role << "\n" << message->content << "<|im_end|>\n";254        }255        if (add_ass) {256            ss << "<|im_start|>assistant\n";257        }258    } else if (tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V7 || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V7_TEKKEN) {259        // Official mistral 'v7' template260        // See: https://huggingface.co/mistralai/Mistral-Large-Instruct-2411#basic-instruct-template-v7261        //      https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503#basic-instruct-template-v7-tekken262        const char * trailing_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V7 ? " " : "";263        for (auto message : chat) {264            std::string role(message->role);265            std::string content(message->content);266            if (role == "system") {267                ss << "[SYSTEM_PROMPT]" << trailing_space << content << "[/SYSTEM_PROMPT]";268            } else if (role == "user") {269                ss << "[INST]" << trailing_space << content << "[/INST]";270            } else {271                ss << trailing_space << content << "</s>";272            }273        }274    } else if (tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V1275            || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3276            || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN) {277        // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/chat_templates.md278        // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/templates.md279        std::string leading_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V1 ? " " : "";280        std::string trailing_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN ? "" : " ";281        bool trim_assistant_message = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3;282        bool is_inside_turn = false;283        for (auto message : chat) {284            if (!is_inside_turn) {285                ss << leading_space << "[INST]" << trailing_space;286                is_inside_turn = true;287            }288            std::string role(message->role);289            std::string content(message->content);290            if (role == "system") {291                ss << content << "\n\n";292            } else if (role == "user") {293                ss << content << leading_space << "[/INST]";294            } else {295                ss << trailing_space << (trim_assistant_message ? trim(content) : content) << "</s>";296                is_inside_turn = false;297            }298        }299    } else if (300            tmpl == LLM_CHAT_TEMPLATE_LLAMA_2301            || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS302            || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS303            || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP) {304        // llama2 template and its variants305        // [variant] support system message306        // See: https://huggingface.co/blog/llama2#how-to-prompt-llama-2307        bool support_system_message = tmpl != LLM_CHAT_TEMPLATE_LLAMA_2;308        // [variant] add BOS inside history309        bool add_bos_inside_history = tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS;310        // [variant] trim spaces from the input message311        bool strip_message = tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP;312        // construct the prompt313        bool is_inside_turn = true; // skip BOS at the beginning314        ss << "[INST] ";315        for (auto message : chat) {316            std::string content = strip_message ? trim(message->content) : message->content;317            std::string role(message->role);318            if (!is_inside_turn) {319                is_inside_turn = true;320                ss << (add_bos_inside_history ? "<s>[INST] " : "[INST] ");321            }322            if (role == "system") {323                if (support_system_message) {324                    ss << "<<SYS>>\n" << content << "\n<</SYS>>\n\n";325                } else {326                    // if the model does not support system message, we still include it in the first message, but without <<SYS>>327                    ss << content << "\n";328                }329            } else if (role == "user") {330                ss << content << " [/INST]";331            } else {332                ss << content << "</s>";333                is_inside_turn = false;334            }335        }336    } else if (tmpl == LLM_CHAT_TEMPLATE_PHI_3) {337        // Phi 3338        for (auto message : chat) {339            std::string role(message->role);340            ss << "<|" << role << "|>\n" << message->content << "<|end|>\n";341        }342        if (add_ass) {343            ss << "<|assistant|>\n";344        }345    } else if (tmpl == LLM_CHAT_TEMPLATE_PHI_4) {346        // chatml template347        for (auto message : chat) {348            ss << "<|im_start|>" << message->role << "<|im_sep|>" << message->content << "<|im_end|>";349        }350        if (add_ass) {351            ss << "<|im_start|>assistant<|im_sep|>";352        }353    } else if (tmpl == LLM_CHAT_TEMPLATE_FALCON_3) {354        // Falcon 3355        for (auto message : chat) {356            std::string role(message->role);357            ss << "<|" << role << "|>\n" << message->content << "\n";358        }359        if (add_ass) {360            ss << "<|assistant|>\n";361        }362    } else if (tmpl == LLM_CHAT_TEMPLATE_ZEPHYR) {363        // zephyr template364        for (auto message : chat) {365            ss << "<|" << message->role << "|>" << "\n" << message->content << "<|endoftext|>\n";366        }367        if (add_ass) {368            ss << "<|assistant|>\n";369        }370    } else if (tmpl == LLM_CHAT_TEMPLATE_MONARCH) {371        // mlabonne/AlphaMonarch-7B template (the <s> is included inside history)372        for (auto message : chat) {373            std::string bos = (message == chat.front()) ? "" : "<s>"; // skip BOS for first message374            ss << bos << message->role << "\n" << message->content << "</s>\n";375        }376        if (add_ass) {377            ss << "<s>assistant\n";378        }379    } else if (tmpl == LLM_CHAT_TEMPLATE_GEMMA) {380        // google/gemma-7b-it381        std::string system_prompt = "";382        for (auto message : chat) {383            std::string role(message->role);384            if (role == "system") {385                // there is no system message for gemma, but we will merge it with user prompt, so nothing is broken386                system_prompt += trim(message->content);387                continue;388            }389            // in gemma, "assistant" is "model"390            role = role == "assistant" ? "model" : message->role;391            ss << "<start_of_turn>" << role << "\n";392            if (!system_prompt.empty() && role != "model") {393                ss << system_prompt << "\n\n";394                system_prompt = "";395            }396            ss << trim(message->content) << "<end_of_turn>\n";397        }398        if (add_ass) {399            ss << "<start_of_turn>model\n";400        }401    } else if (tmpl == LLM_CHAT_TEMPLATE_ORION) {402        // OrionStarAI/Orion-14B-Chat403        std::string system_prompt = "";404        for (auto message : chat) {405            std::string role(message->role);406            if (role == "system") {407                // there is no system message support, we will merge it with user prompt408                system_prompt += message->content;409                continue;410            } else if (role == "user") {411                ss << "Human: ";412                if (!system_prompt.empty()) {413                    ss << system_prompt << "\n\n";414                    system_prompt = "";415                }416                ss << message->content << "\n\nAssistant: </s>";417            } else {418                ss << message->content << "</s>";419            }420        }421    } else if (tmpl == LLM_CHAT_TEMPLATE_OPENCHAT) {422        // openchat/openchat-3.5-0106,423        for (auto message : chat) {424            std::string role(message->role);425            if (role == "system") {426                ss << message->content << "<|end_of_turn|>";427            } else {428                role[0] = toupper(role[0]);429                ss << "GPT4 Correct " << role << ": " << message->content << "<|end_of_turn|>";430            }431        }432        if (add_ass) {433            ss << "GPT4 Correct Assistant:";434        }435    } else if (tmpl == LLM_CHAT_TEMPLATE_VICUNA || tmpl == LLM_CHAT_TEMPLATE_VICUNA_ORCA) {436        // eachadea/vicuna-13b-1.1 (and Orca variant)437        for (auto message : chat) {438            std::string role(message->role);439            if (role == "system") {440                // Orca-Vicuna variant uses a system prefix441                if (tmpl == LLM_CHAT_TEMPLATE_VICUNA_ORCA) {442                    ss << "SYSTEM: " << message->content << "\n";443                } else {444                    ss << message->content << "\n\n";445                }446            } else if (role == "user") {447                ss << "USER: " << message->content << "\n";448            } else if (role == "assistant") {449                ss << "ASSISTANT: " << message->content << "</s>\n";450            }451        }452        if (add_ass) {453            ss << "ASSISTANT:";454        }455    } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK) {456        // deepseek-ai/deepseek-coder-33b-instruct457        for (auto message : chat) {458            std::string role(message->role);459            if (role == "system") {460                ss << message->content;461            } else if (role == "user") {462                ss << "### Instruction:\n" << message->content << "\n";463            } else if (role == "assistant") {464                ss << "### Response:\n" << message->content << "\n<|EOT|>\n";465            }466        }467        if (add_ass) {468            ss << "### Response:\n";469        }470    } else if (tmpl == LLM_CHAT_TEMPLATE_COMMAND_R) {471        // CohereForAI/c4ai-command-r-plus472        for (auto message : chat) {473            std::string role(message->role);474            if (role == "system") {475                ss << "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>" << trim(message->content) << "<|END_OF_TURN_TOKEN|>";476            } else if (role == "user") {477                ss << "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>" << trim(message->content) << "<|END_OF_TURN_TOKEN|>";478            } else if (role == "assistant") {479                ss << "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" << trim(message->content) << "<|END_OF_TURN_TOKEN|>";480            }481        }482        if (add_ass) {483            ss << "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>";484        }485    } else if (tmpl == LLM_CHAT_TEMPLATE_LLAMA_3) {486        // Llama 3487        for (auto message : chat) {488            std::string role(message->role);489            ss << "<|start_header_id|>" << role << "<|end_header_id|>\n\n" << trim(message->content) << "<|eot_id|>";490        }491        if (add_ass) {492            ss << "<|start_header_id|>assistant<|end_header_id|>\n\n";493        }494    } else if (tmpl == LLM_CHAT_TEMPLATE_CHATGLM_3) {495        // chatglm3-6b496        ss << "[gMASK]" << "sop";497        for (auto message : chat) {498            std::string role(message->role);499            ss << "<|" << role << "|>" << "\n " << message->content;500        }501        if (add_ass) {502            ss << "<|assistant|>";503        }504    } else if (tmpl == LLM_CHAT_TEMPLATE_CHATGLM_4) {505        ss << "[gMASK]" << "<sop>";506        for (auto message : chat) {507            std::string role(message->role);508            ss << "<|" << role << "|>" << "\n" << message->content;509        }510        if (add_ass) {511            ss << "<|assistant|>\n";512        }513    } else if (tmpl == LLM_CHAT_TEMPLATE_GLMEDGE) {514        for (auto message : chat) {515            std::string role(message->role);516            ss << "<|" << role << "|>" << "\n" << message->content;517        }518        if (add_ass) {519            ss << "<|assistant|>";520        }521    } else if (tmpl == LLM_CHAT_TEMPLATE_MINICPM) {522        // MiniCPM-3B-OpenHermes-2.5-v2-GGUF523        for (auto message : chat) {524            std::string role(message->role);525            if (role == "user") {526                ss << LU8("<用户>");527                ss << trim(message->content);528                ss << "<AI>";529            } else {530                ss << trim(message->content);531            }532        }533    } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_2) {534        // DeepSeek-V2535        for (auto message : chat) {536            std::string role(message->role);537            if (role == "system") {538                ss << message->content << "\n\n";539            } else if (role == "user") {540                ss << "User: " << message->content << "\n\n";541            } else if (role == "assistant") {542                ss << "Assistant: " << message->content << LU8("<｜end▁of▁sentence｜>");543            }544        }545        if (add_ass) {546            ss << "Assistant:";547        }548    } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_3) {549        // DeepSeek-V3550        for (auto message : chat) {551            std::string role(message->role);552            if (role == "system") {553                ss << message->content << "\n\n";554            } else if (role == "user") {555                ss << LU8("<｜User｜>") << message->content;556            } else if (role == "assistant") {557                ss << LU8("<｜Assistant｜>") << message->content << LU8("<｜end▁of▁sentence｜>");558            }559        }560        if (add_ass) {561            ss << LU8("<｜Assistant｜>");562        }563    } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_OCR) {564        for (auto message : chat) {565            // no template566            ss << message->content;567        }568    } else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_3) {569        // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb570        // EXAONE-3.0-7.8B-Instruct571        for (auto message : chat) {572            std::string role(message->role);573            if (role == "system") {574                ss << "[|system|]" << trim(message->content) << "[|endofturn|]\n";575            } else if (role == "user") {576                ss << "[|user|]" << trim(message->content) << "\n";577            } else if (role == "assistant") {578                ss << "[|assistant|]" << trim(message->content) << "[|endofturn|]\n";579            }580        }581        if (add_ass) {582            ss << "[|assistant|]";583        }584    } else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_4) {585        for (auto message : chat) {586            std::string role(message->role);587            if (role == "system") {588                ss << "[|system|]" << trim(message->content) << "[|endofturn|]\n";589            } else if (role == "user") {590                ss << "[|user|]" << trim(message->content) << "\n";591            } else if (role == "assistant") {592                ss << "[|assistant|]" << trim(message->content) << "[|endofturn|]\n";593            } else if (role == "tool") {594                ss << "[|tool|]" << trim(message->content) << "[|endofturn|]\n";595            }596        }597        if (add_ass) {598            ss << "[|assistant|]";599        }600    } else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_MOE) {601        for (auto message : chat) {602            std::string role(message->role);603            if (role == "system") {604                ss << "<|system|>\n" << trim(message->content) << "<|endofturn|>\n";605            } else if (role == "user") {606                ss << "<|user|>\n" << trim(message->content) << "<|endofturn|>\n";607            } else if (role == "assistant") {608                ss << "<|assistant|>\n" << trim(message->content) << "<|endofturn|>\n";609            } else if (role == "tool") {610                ss << "<|tool|>\n" << trim(message->content) << "<|endofturn|>\n";611            }612        }613        if (add_ass) {614            ss << "<|assistant|>\n";615        }616    } else if (tmpl == LLM_CHAT_TEMPLATE_RWKV_WORLD) {617        // this template requires the model to have "\n\n" as EOT token618        for (size_t i = 0; i < chat.size(); i++) {619            std::string role(chat[i]->role);620            if (role == "system") {621                ss << "System: " << trim(chat[i]->content) << "\n\n";622            } else if (role == "user") {623                ss << "User: " << trim(chat[i]->content) << "\n\n";624                if (i == chat.size() - 1) {625                    ss << "Assistant:";626                }627            } else if (role == "assistant") {628                ss << "Assistant: " << trim(chat[i]->content) << "\n\n";629            }630        }631    } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE_3_X) {632        // IBM Granite 3.x template633        for (const auto & message : chat) {634            std::string role(message->role);635            ss << "<|start_of_role|>" << role << "<|end_of_role|>";636            if (role == "assistant_tool_call") {637                ss << "<|tool_call|>";638            }639            ss << message->content << "<|end_of_text|>\n";640        }641        if (add_ass) {642            ss << "<|start_of_role|>assistant<|end_of_role|>";643        }644    } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE_4_0) {645        // IBM Granite 4.0 template646        for (const auto & message : chat) {647            std::string role(message->role);648            if (role == "assistant_tool_call") {649                ss << "<|start_of_role|>assistant<|end_of_role|><|tool_call|>";650            } else {651                ss << "<|start_of_role|>" << role << "<|end_of_role|>";652            }653            ss << message->content << "<|end_of_text|>\n";654        }655        if (add_ass) {656            ss << "<|start_of_role|>assistant<|end_of_role|>";657        }658    } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE_4_1) {659        // IBM Granite 4.1 template660        for (const auto & message : chat) {661            std::string role(message->role);662            if (role == "assistant_tool_call") {663                ss << "<|start_of_role|>assistant<|end_of_role|><|tool_call|>";664            } else {665                ss << "<|start_of_role|>" << role << "<|end_of_role|>";666            }667            ss << message->content << "<|end_of_text|>\n";668        }669        if (add_ass) {670            ss << "<|start_of_role|>assistant<|end_of_role|>";671        }672    } else if (tmpl == LLM_CHAT_TEMPLATE_GIGACHAT) {673        // GigaChat template674        bool has_system = !chat.empty() && std::string(chat[0]->role) == "system";675 676        // Handle system message if present677        if (has_system) {678            ss << "<s>" << chat[0]->content << "<|message_sep|>";679        } else {680            ss << "<s>";681        }682 683        // Process remaining messages684        for (size_t i = has_system ? 1 : 0; i < chat.size(); i++) {685            std::string role(chat[i]->role);686            if (role == "user") {687                ss << "user<|role_sep|>" << chat[i]->content << "<|message_sep|>"688                << "available functions<|role_sep|>[]<|message_sep|>";689            } else if (role == "assistant") {690                ss << "assistant<|role_sep|>" << chat[i]->content << "<|message_sep|>";691            }692        }693 694        // Add generation prompt if needed695        if (add_ass) {696            ss << "assistant<|role_sep|>";697        }698    }  else if (tmpl == LLM_CHAT_TEMPLATE_MEGREZ) {699        // Megrez template700        for (auto message : chat) {701            std::string role(message->role);702            ss << "<|role_start|>" << role << "<|role_end|>" << message->content << "<|turn_end|>";703        }704 705        if (add_ass) {706            ss << "<|role_start|>assistant<|role_end|>";707        }708    } else if (tmpl == LLM_CHAT_TEMPLATE_YANDEX) {709        // Yandex template ("\n\n" is defined as EOT token)710 711        for (size_t i = 0; i < chat.size(); i++) {712            std::string role(chat[i]->role);713            if (role == "user") {714                ss << " Пользователь: " << chat[i]->content << "\n\n";715            } else if (role == "assistant") {716                ss << " Ассистент: " << chat[i]->content << "\n\n";717            }718        }719 720        // Add generation prompt if needed721        if (add_ass) {722            ss << " Ассистент:[SEP]";723        }724    } else if (tmpl == LLM_CHAT_TEMPLATE_BAILING || tmpl == LLM_CHAT_TEMPLATE_BAILING_THINK) {725        // Bailing (Ling/Ring) template726        for (auto message : chat) {727            std::string role(message->role);728 729            if (role == "user") {730                role = "HUMAN";731            } else {732                std::transform(role.begin(), role.end(), role.begin(), ::toupper);733            }734 735            ss << "<role>" << role << "</role>" << message->content;736        }737 738        if (add_ass) {739            ss << "<role>ASSISTANT</role>";740 741            if (tmpl == LLM_CHAT_TEMPLATE_BAILING_THINK) {742                ss << "<think>";743            }744        }745    } else if (tmpl == LLM_CHAT_TEMPLATE_BAILING2) {746        // Bailing2 (Ling 2.0) template747        bool has_system = !chat.empty() && std::string(chat[0]->role) == "system";748 749        if (!has_system) {750            ss << "<role>SYSTEM</role>detailed thinking off<|role_end|>";751        }752 753        for (auto message : chat) {754            std::string role(message->role);755 756            if (role == "user") {757                role = "HUMAN";758            } else {759                std::transform(role.begin(), role.end(), role.begin(), ::toupper);760            }761 762            ss << "<role>" << role << "</role>" << message->content << "<|role_end|>";763        }764 765        if (add_ass) {766            ss << "<role>ASSISTANT</role>";767        }768    } else if (tmpl == LLM_CHAT_TEMPLATE_LLAMA4) {769        // Llama 4770        for (auto message : chat) {771            std::string role(message->role);772            ss << "<|header_start|>" << role << "<|header_end|>\n\n" << trim(message->content) << "<|eot|>";773        }774        if (add_ass) {775            ss << "<|header_start|>assistant<|header_end|>\n\n";776        }777    } else if (tmpl == LLM_CHAT_TEMPLATE_SMOLVLM) {778        // SmolVLM779        ss << "<|im_start|>"; // uses <|im_start|> as BOS, but the actual content is NOT chatml780        for (auto message : chat) {781            std::string role(message->role);782            if (role == "system") {783                ss << message->content << "\n\n";784            } else if (role == "user") {785                ss << "User: " << message->content << "<end_of_utterance>\n";786            } else {787                ss << "Assistant: " << message->content << "<end_of_utterance>\n";788            }789        }790        if (add_ass) {791            ss << "Assistant:";792        }793    } else if (tmpl == LLM_CHAT_TEMPLATE_DOTS1) {794        // dots.llm1.inst (DOTS1)795        for (auto message : chat) {796            std::string role(message->role);797            if (role == "system") {798                ss << "<|system|>" << message->content << "<|endofsystem|>";799            } else if (role == "user") {800                ss << "<|userprompt|>" << message->content << "<|endofuserprompt|>";801            } else {802                ss << "<|response|>" << message->content << "<|endofresponse|>";803            }804        }805        if (add_ass) {806            ss << "<|response|>";807        }808    } else if (tmpl == LLM_CHAT_TEMPLATE_HUNYUAN_MOE) {809        // tencent/Hunyuan-A13B-Instruct810        for (auto message : chat) {811            std::string role(message->role);812            if (role == "system") {813                ss << "<|startoftext|>" << message->content << "<|extra_4|>";814            } else if (role == "assistant") {815                ss << message->content << "<|eos|>";816            } else {817                ss << "<|startoftext|>" << message->content << "<|extra_0|>";818            }819        }820    } else if (tmpl == LLM_CHAT_TEMPLATE_OPENAI_MOE) {821        // OpenAI MoE (based on Harmony chat template)822        for (auto message : chat) {823            std::string role(message->role);824            ss << "<|start|>" << role << "<|message|>" << message->content;825            ss << (role == "assistant" ? "<|return|>" : "<|end|>");826        }827        if (add_ass) {828            ss << "<|start|>assistant";829        }830    } else if (tmpl == LLM_CHAT_TEMPLATE_HUNYUAN_DENSE) {831        // tencent/Hunyuan-4B-Instruct832        for (size_t i = 0; i < chat.size(); i++) {833            std::string role(chat[i]->role);834            if (i == 0) {835                if (role == "system") {836                    ss << chat[i]->content << "<｜hy_place▁holder▁no▁3｜>";837                }838            }839 840            if (role == "assistant") {841                ss << "<｜hy_Assistant｜>" << chat[i]->content << "<｜hy_place▁holder▁no▁2｜>";842            } else if (role == "user") {843                ss << "<｜hy_User｜>" << chat[i]->content << "<｜hy_Assistant｜>";844            }845        }846    } else if (tmpl == LLM_CHAT_TEMPLATE_HUNYUAN_VL) {847        // tencent/HunyuanOCR & tencent/HunyuanVL848        ss << "<｜hy_begin▁of▁sentence｜>";849        for (size_t i = 0; i < chat.size(); i++) {850            std::string role(chat[i]->role);851            if (i == 0 && role == "system") {852                ss << chat[i]->content << "<｜hy_place▁holder▁no▁3｜>";853                continue;854            }855 856            if (role == "user") {857                ss << chat[i]->content << "<｜hy_User｜>";858            } else if (role == "assistant") {859                ss << chat[i]->content << "<｜hy_Assistant｜>";860            }861        }862    } else if (tmpl == LLM_CHAT_TEMPLATE_KIMI_K2) {863        // moonshotai/Kimi-K2-Instruct864        for (auto message : chat) {865            std::string role(message->role);866            if (role == "system") {867                ss << "<|im_system|>system<|im_middle|>";868            } else if (role == "user") {869                ss << "<|im_user|>user<|im_middle|>";870            } else if (role == "assistant") {871                ss << "<|im_assistant|>assistant<|im_middle|>";872            } else if (role == "tool") {873                ss << "<|im_system|>tool<|im_middle|>";874            }875 876            ss << message->content << "<|im_end|>";877        }878        if (add_ass) {879            ss << "<|im_assistant|>assistant<|im_middle|>";880        }881    } else if (tmpl == LLM_CHAT_TEMPLATE_SEED_OSS) {882        for (auto message: chat) {883            std::string role(message->role);884            ss << "<seed:bos>" << role << "\n" << (role == "assistant" ? trim(message->content) : message->content) << "<seed:eos>";885        }886        if (add_ass) {887            ss << "<seed:bos>assistant\n";888        }889    } else if (tmpl == LLM_CHAT_TEMPLATE_GROK_2) {890        for (auto message : chat) {891            std::string role(message->role);892            if (role == "system") {893                ss << "System: " << trim(message->content) << "<|separator|>\n\n";894            } else if (role == "user") {895                ss << "Human: " << trim(message->content) << "<|separator|>\n\n";896            } else if (role == "assistant") {897                ss << "Assistant: " << message->content << "<|separator|>\n\n";898            }899        }900        if (add_ass) {901            ss << "Assistant:";902        }903    }else if (tmpl == LLM_CHAT_TEMPLATE_PANGU_EMBED) {904        // [unused9]系统：xxx[unused10]905        // [unused9]用户：xxx[unused10]906        // [unused9]助手：xxx[unused10]907        // ...908        for (size_t i = 0; i < chat.size(); ++i) {909            const auto & msg = chat[i];910            const std::string & role = msg->role;911            const std::string & content = msg->content;912 913            if (i == 0 && role != "system") {914                ss << "[unused9]系统：[unused10]";915            }916 917            if (role == "system") {918                ss << "[unused9]系统：" << content << "[unused10]";919            } else if (role == "user") {920                ss << "[unused9]用户：" << content << "[unused10]";921            } else if (role == "assistant") {922                ss << "[unused9]助手：" << content << "[unused10]";923            } else if (role == "tool") {924                ss << "[unused9]工具：" << content << "[unused10]";925            } else if (role == "function") {926                ss << "[unused9]方法：" << content << "[unused10]";927            }928        }929        if (add_ass) {930            ss << "[unused9]助手：";931        }932    } else if (tmpl == LLM_CHAT_TEMPLATE_SOLAR_OPEN) {933        for (auto message : chat) {934            std::string role(message->role);935            ss << "<|begin|>" << role << "<|content|>" << message->content << "<|end|>";936        }937        if (add_ass) {938            ss << "<|begin|>assistant";939        }940    } else {941        // template not supported942        return -1;943    }944    dest = ss.str();945    return dest.size();946}947 948// public interface949 950int32_t llama_chat_builtin_templates(const char ** output, size_t len) {951    auto it = LLM_CHAT_TEMPLATES.begin();952    for (size_t i = 0; i < std::min(len, LLM_CHAT_TEMPLATES.size()); i++) {953        output[i] = it->first.c_str();954        std::advance(it, 1);955    }956    return (int32_t) LLM_CHAT_TEMPLATES.size();957}958