echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0762
1#include "server-task.h"2 3#include "build-info.h"4#include "chat.h"5#include "common.h"6#include "json-schema-to-grammar.h"7#include "llama.h"8#include "sampling.h"9#include "speculative.h"10#include "server-common.h"11 12using json = nlohmann::ordered_json;13 14//15// task_params16//17 18json task_params::format_logit_bias(const std::vector<llama_logit_bias> & logit_bias) const {19 json data = json::array();20 for (const auto & lb : logit_bias) {21 data.push_back(json{22 {"bias", lb.bias},23 {"token", lb.token},24 });25 }26 return data;27}28 29json task_params::to_json(bool only_metrics) const {30 std::vector<std::string> samplers;31 samplers.reserve(sampling.samplers.size());32 for (const auto & sampler : sampling.samplers) {33 samplers.emplace_back(common_sampler_type_to_str(sampler));34 }35 36 json lora = json::array();37 for (auto & it : this->lora) {38 lora.push_back({{"id", it.first}, {"scale", it.second}});39 }40 41 if (only_metrics) {42 return json {43 {"seed", sampling.seed},44 {"temperature", sampling.temp},45 {"dynatemp_range", sampling.dynatemp_range},46 {"dynatemp_exponent", sampling.dynatemp_exponent},47 {"top_k", sampling.top_k},48 {"top_p", sampling.top_p},49 {"min_p", sampling.min_p},50 {"top_n_sigma", sampling.top_n_sigma},51 {"xtc_probability", sampling.xtc_probability},52 {"xtc_threshold", sampling.xtc_threshold},53 {"typical_p", sampling.typ_p},54 {"repeat_last_n", sampling.penalty_last_n},55 {"repeat_penalty", sampling.penalty_repeat},56 {"presence_penalty", sampling.penalty_present},57 {"frequency_penalty", sampling.penalty_freq},58 {"dry_multiplier", sampling.dry_multiplier},59 {"dry_base", sampling.dry_base},60 {"dry_allowed_length", sampling.dry_allowed_length},61 {"dry_penalty_last_n", sampling.dry_penalty_last_n},62 {"mirostat", sampling.mirostat},63 {"mirostat_tau", sampling.mirostat_tau},64 {"mirostat_eta", sampling.mirostat_eta},65 {"max_tokens", n_predict},66 {"n_predict", n_predict}, // TODO: deduplicate?67 {"n_keep", n_keep},68 {"n_discard", n_discard},69 {"ignore_eos", sampling.ignore_eos},70 {"stream", stream},71 {"n_probs", sampling.n_probs},72 {"min_keep", sampling.min_keep},73 {"chat_format", common_chat_format_name(chat_parser_params.format)},74 {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},75 {"reasoning_in_content", chat_parser_params.reasoning_in_content},76 {"generation_prompt", chat_parser_params.generation_prompt},77 {"samplers", samplers},78 {"speculative.n_max", speculative.n_max},79 {"speculative.n_min", speculative.n_min},80 {"speculative.p_min", speculative.p_min},81 {"speculative.type", common_speculative_type_to_str(speculative.type)},82 {"speculative.ngram_size_n", speculative.ngram_size_n},83 {"speculative.ngram_size_m", speculative.ngram_size_m},84 {"speculative.ngram_m_hits", speculative.ngram_min_hits},85 {"timings_per_token", timings_per_token},86 {"post_sampling_probs", post_sampling_probs},87 {"backend_sampling", sampling.backend_sampling},88 {"lora", lora},89 };90 }91 92 auto grammar_triggers = json::array();93 for (const auto & trigger : sampling.grammar_triggers) {94 server_grammar_trigger ct(trigger);95 grammar_triggers.push_back(ct.to_json());96 }97 98 return json {99 {"seed", sampling.seed},100 {"temperature", sampling.temp},101 {"dynatemp_range", sampling.dynatemp_range},102 {"dynatemp_exponent", sampling.dynatemp_exponent},103 {"top_k", sampling.top_k},104 {"top_p", sampling.top_p},105 {"min_p", sampling.min_p},106 {"top_n_sigma", sampling.top_n_sigma},107 {"xtc_probability", sampling.xtc_probability},108 {"xtc_threshold", sampling.xtc_threshold},109 {"typical_p", sampling.typ_p},110 {"repeat_last_n", sampling.penalty_last_n},111 {"repeat_penalty", sampling.penalty_repeat},112 {"presence_penalty", sampling.penalty_present},113 {"frequency_penalty", sampling.penalty_freq},114 {"dry_multiplier", sampling.dry_multiplier},115 {"dry_base", sampling.dry_base},116 {"dry_allowed_length", sampling.dry_allowed_length},117 {"dry_penalty_last_n", sampling.dry_penalty_last_n},118 {"dry_sequence_breakers", sampling.dry_sequence_breakers},119 {"mirostat", sampling.mirostat},120 {"mirostat_tau", sampling.mirostat_tau},121 {"mirostat_eta", sampling.mirostat_eta},122 {"stop", antiprompt},123 {"max_tokens", n_predict},124 {"n_predict", n_predict}, // TODO: deduplicate?125 {"n_keep", n_keep},126 {"n_discard", n_discard},127 {"ignore_eos", sampling.ignore_eos},128 {"stream", stream},129 {"logit_bias", format_logit_bias(sampling.logit_bias)},130 {"n_probs", sampling.n_probs},131 {"min_keep", sampling.min_keep},132 {"grammar", common_grammar_value(sampling.grammar)},133 {"grammar_lazy", sampling.grammar_lazy},134 {"grammar_triggers", grammar_triggers},135 {"preserved_tokens", sampling.preserved_tokens},136 {"chat_format", common_chat_format_name(chat_parser_params.format)},137 {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},138 {"reasoning_in_content", chat_parser_params.reasoning_in_content},139 {"generation_prompt", chat_parser_params.generation_prompt},140 {"samplers", samplers},141 {"speculative.n_max", speculative.n_max},142 {"speculative.n_min", speculative.n_min},143 {"speculative.p_min", speculative.p_min},144 {"speculative.type", common_speculative_type_to_str(speculative.type)},145 {"speculative.ngram_size_n", speculative.ngram_size_n},146 {"speculative.ngram_size_m", speculative.ngram_size_m},147 {"speculative.ngram_m_hits", speculative.ngram_min_hits},148 {"timings_per_token", timings_per_token},149 {"post_sampling_probs", post_sampling_probs},150 {"backend_sampling", sampling.backend_sampling},151 {"lora", lora},152 };153}154 155//156// task_result_state157//158common_chat_msg task_result_state::update_chat_msg(159 const std::string & text_added,160 bool is_partial,161 std::vector<common_chat_msg_diff> & diffs,162 bool filter_tool_calls) {163 generated_text += text_added;164 auto msg_prv_copy = chat_msg;165 //SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());166 auto new_msg = common_chat_parse(167 generated_text,168 is_partial,169 chat_parser_params);170 if (!new_msg.empty()) {171 new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id);172 chat_msg = new_msg;173 auto all_diffs = common_chat_msg_diff::compute_diffs(msg_prv_copy, chat_msg);174 175 if (!filter_tool_calls) {176 diffs = std::move(all_diffs);177 } else {178 for (auto & d : all_diffs) {179 // If this is a new type of delta, flush all currently pending tool call names180 for (size_t i = 0; i < chat_msg.tool_calls.size(); ++i) {181 if (sent_tool_call_names.count(i) || chat_msg.tool_calls[i].name.empty()) {182 continue;183 }184 if (d.tool_call_index != i || !d.tool_call_delta.arguments.empty()) {185 common_chat_msg_diff header;186 header.tool_call_index = i;187 header.tool_call_delta.id = chat_msg.tool_calls[i].id;188 header.tool_call_delta.name = chat_msg.tool_calls[i].name;189 diffs.push_back(std::move(header));190 sent_tool_call_names.insert(i);191 }192 }193 194 if (d.tool_call_index == std::string::npos) {195 diffs.push_back(std::move(d));196 } else {197 size_t i = d.tool_call_index;198 if (sent_tool_call_names.count(i)) {199 if (!d.tool_call_delta.arguments.empty()) {200 d.tool_call_delta.name = "";201 d.tool_call_delta.id = "";202 diffs.push_back(std::move(d));203 }204 } else {205 // Not sent yet.206 if (!d.tool_call_delta.arguments.empty() || !is_partial) {207 d.tool_call_delta.name = chat_msg.tool_calls[i].name;208 d.tool_call_delta.id = chat_msg.tool_calls[i].id;209 diffs.push_back(std::move(d));210 sent_tool_call_names.insert(i);211 } else {212 // Suppress213 }214 }215 }216 }217 // Final check at EOF218 if (!is_partial) {219 for (size_t i = 0; i < chat_msg.tool_calls.size(); ++i) {220 if (!sent_tool_call_names.count(i) && !chat_msg.tool_calls[i].name.empty()) {221 common_chat_msg_diff header;222 header.tool_call_index = i;223 header.tool_call_delta.id = chat_msg.tool_calls[i].id;224 header.tool_call_delta.name = chat_msg.tool_calls[i].name;225 diffs.push_back(std::move(header));226 sent_tool_call_names.insert(i);227 }228 }229 }230 }231 }232 return chat_msg;233}234 235//236// server_task237//238 239task_params server_task::params_from_json_cmpl(240 const llama_vocab * vocab,241 const common_params & params_base,242 const int n_ctx_slot,243 const std::vector<llama_logit_bias> & logit_bias_eog,244 const json & data) {245 task_params params;246 247 // Sampling parameter defaults are loaded from the global server context (but individual requests can still them)248 task_params defaults;249 defaults.sampling = params_base.sampling;250 defaults.speculative = params_base.speculative;251 defaults.n_keep = params_base.n_keep;252 defaults.n_predict = params_base.n_predict;253 defaults.n_cache_reuse = params_base.n_cache_reuse;254 defaults.cache_prompt = params_base.cache_prompt;255 defaults.antiprompt = params_base.antiprompt;256 257 // enabling this will output extra debug information in the HTTP responses from the server258 params.verbose = params_base.verbosity > 9;259 params.timings_per_token = json_value(data, "timings_per_token", false);260 261 params.stream = json_value(data, "stream", false);262 auto stream_opt = json_value(data, "stream_options", json::object());263 params.include_usage = json_value(stream_opt, "include_usage", false);264 params.cache_prompt = json_value(data, "cache_prompt", defaults.cache_prompt);265 params.return_tokens = json_value(data, "return_tokens", false);266 params.return_progress = json_value(data, "return_progress", false);267 auto max_tokens = json_value(data, "max_tokens", defaults.n_predict);268 params.n_predict = json_value(data, "n_predict", json_value(data, "max_completion_tokens", max_tokens));269 params.n_indent = json_value(data, "n_indent", defaults.n_indent);270 params.n_keep = json_value(data, "n_keep", defaults.n_keep);271 params.n_discard = json_value(data, "n_discard", defaults.n_discard);272 params.n_cmpl = json_value(data, "n_cmpl", json_value(data, "n", 1));273 params.n_cache_reuse = json_value(data, "n_cache_reuse", defaults.n_cache_reuse);274 //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement275 params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms);276 params.response_fields = json_value(data, "response_fields", std::vector<std::string>());277 278 params.sampling.top_k = json_value(data, "top_k", defaults.sampling.top_k);279 params.sampling.top_p = json_value(data, "top_p", defaults.sampling.top_p);280 params.sampling.min_p = json_value(data, "min_p", defaults.sampling.min_p);281 params.sampling.top_n_sigma = json_value(data, "top_n_sigma", defaults.sampling.top_n_sigma);282 params.sampling.xtc_probability = json_value(data, "xtc_probability", defaults.sampling.xtc_probability);283 params.sampling.xtc_threshold = json_value(data, "xtc_threshold", defaults.sampling.xtc_threshold);284 params.sampling.typ_p = json_value(data, "typical_p", defaults.sampling.typ_p);285 params.sampling.temp = json_value(data, "temperature", defaults.sampling.temp);286 params.sampling.dynatemp_range = json_value(data, "dynatemp_range", defaults.sampling.dynatemp_range);287 params.sampling.dynatemp_exponent = json_value(data, "dynatemp_exponent", defaults.sampling.dynatemp_exponent);288 params.sampling.penalty_last_n = json_value(data, "repeat_last_n", defaults.sampling.penalty_last_n);289 params.sampling.penalty_repeat = json_value(data, "repeat_penalty", defaults.sampling.penalty_repeat);290 params.sampling.penalty_freq = json_value(data, "frequency_penalty", defaults.sampling.penalty_freq);291 params.sampling.penalty_present = json_value(data, "presence_penalty", defaults.sampling.penalty_present);292 params.sampling.dry_multiplier = json_value(data, "dry_multiplier", defaults.sampling.dry_multiplier);293 params.sampling.dry_base = json_value(data, "dry_base", defaults.sampling.dry_base);294 params.sampling.dry_allowed_length = json_value(data, "dry_allowed_length", defaults.sampling.dry_allowed_length);295 params.sampling.dry_penalty_last_n = json_value(data, "dry_penalty_last_n", defaults.sampling.dry_penalty_last_n);296 params.sampling.mirostat = json_value(data, "mirostat", defaults.sampling.mirostat);297 params.sampling.mirostat_tau = json_value(data, "mirostat_tau", defaults.sampling.mirostat_tau);298 params.sampling.mirostat_eta = json_value(data, "mirostat_eta", defaults.sampling.mirostat_eta);299 params.sampling.adaptive_target = json_value(data, "adaptive_target", defaults.sampling.adaptive_target);300 params.sampling.adaptive_decay = json_value(data, "adaptive_decay", defaults.sampling.adaptive_decay);301 params.sampling.seed = json_value(data, "seed", defaults.sampling.seed);302 params.sampling.n_probs = json_value(data, "n_probs", defaults.sampling.n_probs);303 params.sampling.min_keep = json_value(data, "min_keep", defaults.sampling.min_keep);304 params.sampling.backend_sampling = json_value(data, "backend_sampling", defaults.sampling.backend_sampling);305 params.post_sampling_probs = json_value(data, "post_sampling_probs", defaults.post_sampling_probs);306 307 params.speculative = defaults.speculative;308 309 params.speculative.n_min = json_value(data, "speculative.n_min", defaults.speculative.n_min);310 params.speculative.n_max = json_value(data, "speculative.n_max", defaults.speculative.n_max);311 params.speculative.p_min = json_value(data, "speculative.p_min", defaults.speculative.p_min);312 313 params.speculative.n_min = std::min(params.speculative.n_max, params.speculative.n_min);314 params.speculative.n_min = std::max(params.speculative.n_min, 0);315 params.speculative.n_max = std::max(params.speculative.n_max, 0);316 317 params.speculative.type = common_speculative_type_from_name(json_value(data, "speculative.type", common_speculative_type_to_str(defaults.speculative.type)));318 319 params.speculative.ngram_size_n = json_value(data, "speculative.ngram_size_n", defaults.speculative.ngram_size_n);320 params.speculative.ngram_size_m = json_value(data, "speculative.ngram_size_m", defaults.speculative.ngram_size_m);321 params.speculative.ngram_min_hits = json_value(data, "speculative.ngram_m_hits", defaults.speculative.ngram_min_hits);322 323 params.speculative.ngram_size_n = std::max(std::min(1, (int) params.speculative.ngram_size_n), 1024);324 params.speculative.ngram_size_m = std::max(std::min(1, (int) params.speculative.ngram_size_m), 1024);325 params.speculative.ngram_min_hits = std::max(std::min(1, (int) params.speculative.ngram_min_hits), 1024);326 327 // Use OpenAI API logprobs only if n_probs wasn't provided328 if (data.contains("logprobs") && params.sampling.n_probs == defaults.sampling.n_probs){329 params.sampling.n_probs = json_value(data, "logprobs", defaults.sampling.n_probs);330 }331 332 if (data.contains("lora")) {333 if (data.at("lora").is_array()) {334 params.lora = parse_lora_request(data.at("lora"));335 } else {336 throw std::runtime_error("Error: 'lora' must be an array of objects with 'id' and 'scale' fields");337 }338 } else {339 params.lora = {};340 }341 342 // TODO: add more sanity checks for the input parameters343 344 if (params.sampling.penalty_last_n < -1) {345 throw std::runtime_error("Error: repeat_last_n must be >= -1");346 }347 348 if (params.sampling.dry_penalty_last_n < -1) {349 throw std::runtime_error("Error: dry_penalty_last_n must be >= -1");350 }351 352 if (params.sampling.penalty_last_n == -1) {353 // note: should be the slot's context and not the full context, but it's ok354 params.sampling.penalty_last_n = n_ctx_slot;355 }356 357 if (params.sampling.dry_penalty_last_n == -1) {358 params.sampling.dry_penalty_last_n = n_ctx_slot;359 }360 361 if (params.sampling.dry_base < 1.0f) {362 params.sampling.dry_base = defaults.sampling.dry_base;363 }364 365 // sequence breakers for DRY366 {367 // Currently, this is not compatible with TextGen WebUI, Koboldcpp and SillyTavern format368 // Ref: https://github.com/oobabooga/text-generation-webui/blob/d1af7a41ade7bd3c3a463bfa640725edb818ebaf/extensions/openai/typing.py#L39369 370 if (data.contains("dry_sequence_breakers")) {371 params.sampling.dry_sequence_breakers = json_value(data, "dry_sequence_breakers", std::vector<std::string>());372 if (params.sampling.dry_sequence_breakers.empty()) {373 throw std::runtime_error("Error: dry_sequence_breakers must be a non-empty array of strings");374 }375 }376 }377 378 // process "json_schema" and "grammar"379 if (data.contains("json_schema") && !data.contains("grammar")) {380 try {381 auto schema = json_value(data, "json_schema", json::object());382 SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str());383 std::string grammar_str = json_schema_to_grammar(schema);384 SRV_DBG("Converted grammar: %s\n", grammar_str.c_str());385 params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, std::move(grammar_str)};386 } catch (const std::exception & e) {387 throw std::runtime_error(std::string("\"json_schema\": ") + e.what());388 }389 } else {390 params.sampling.grammar = defaults.sampling.grammar;391 392 std::string grammar_str = json_value(data, "grammar", std::string());393 if (!grammar_str.empty()) {394 // grammar_type key is set by the server when converting chat template grammars395 std::string grammar_type = json_value(data, "grammar_type", std::string());396 if (grammar_type == "tool_calls") {397 params.sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, std::move(grammar_str)};398 } else {399 // explicit grammar from the user (API field "grammar")400 params.sampling.grammar = {COMMON_GRAMMAR_TYPE_USER, std::move(grammar_str)};401 }402 SRV_DBG("Grammar (%s): %s\n", grammar_type.c_str(), common_grammar_value(params.sampling.grammar).c_str());403 }404 params.sampling.grammar_lazy = json_value(data, "grammar_lazy", defaults.sampling.grammar_lazy);405 SRV_DBG("Grammar lazy: %s\n", params.sampling.grammar_lazy ? "true" : "false");406 }407 408 {409 auto it = data.find("chat_format");410 if (it != data.end()) {411 params.chat_parser_params.format = static_cast<common_chat_format>(it->get<int>());412 SRV_INF("Chat format: %s\n", common_chat_format_name(params.chat_parser_params.format));413 } else {414 params.chat_parser_params.format = defaults.chat_parser_params.format;415 }416 common_reasoning_format reasoning_format = params_base.reasoning_format;417 if (data.contains("reasoning_format")) {418 reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());419 }420 params.chat_parser_params.reasoning_format = reasoning_format;421 params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);422 params.chat_parser_params.generation_prompt = json_value(data, "generation_prompt", std::string());423 params.sampling.generation_prompt = params.chat_parser_params.generation_prompt;424 SRV_DBG("Generation prompt: '%s'\n", params.chat_parser_params.generation_prompt.c_str());425 params.chat_parser_params.parse_tool_calls = json_value(data, "parse_tool_calls", false);426 if (data.contains("chat_parser")) {427 params.chat_parser_params.parser.load(data.at("chat_parser").get<std::string>());428 }429 }430 431 {432 const auto preserved_tokens = data.find("preserved_tokens");433 if (preserved_tokens != data.end()) {434 for (const auto & t : *preserved_tokens) {435 auto ids = common_tokenize(vocab, t.get<std::string>(), /* add_special= */ false, /* parse_special= */ true);436 if (ids.size() == 1) {437 SRV_DBG("Preserved token: %d\n", ids[0]);438 params.sampling.preserved_tokens.insert(ids[0]);439 } else {440 // This may happen when using a tool call style meant for a model with special tokens to preserve on a model without said tokens.441 SRV_DBG("Not preserved because more than 1 token: %s\n", t.get<std::string>().c_str());442 }443 }444 }445 const auto grammar_triggers = data.find("grammar_triggers");446 if (grammar_triggers != data.end()) {447 for (const auto & t : *grammar_triggers) {448 server_grammar_trigger ct(t);449 if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {450 const auto & word = ct.value.value;451 auto ids = common_tokenize(vocab, word, /* add_special= */ false, /* parse_special= */ true);452 if (ids.size() == 1) {453 auto token = ids[0];454 if (std::find(params.sampling.preserved_tokens.begin(), params.sampling.preserved_tokens.end(), (llama_token) token) == params.sampling.preserved_tokens.end()) {455 throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);456 }457 SRV_DBG("Grammar trigger token: %d (`%s`)\n", token, word.c_str());458 common_grammar_trigger trigger;459 trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;460 trigger.value = word;461 trigger.token = token;462 params.sampling.grammar_triggers.push_back(std::move(trigger));463 } else {464 SRV_DBG("Grammar trigger word: `%s`\n", word.c_str());465 params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});466 }467 } else {468 if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN) {469 SRV_DBG("Grammar trigger pattern: `%s`\n", ct.value.value.c_str());470 } else if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL) {471 SRV_DBG("Grammar trigger pattern full: `%s`\n", ct.value.value.c_str());472 } else {473 throw std::runtime_error("Unknown grammar trigger type");474 }475 params.sampling.grammar_triggers.emplace_back(std::move(ct.value));476 }477 }478 }479 if (params.sampling.grammar_lazy && params.sampling.grammar_triggers.empty()) {480 throw std::runtime_error("Error: no triggers set for lazy grammar!");481 }482 }483 484 // Parse reasoning budget sampler parameters485 {486 const int32_t budget = json_value(data, "reasoning_budget_tokens", (int32_t) -1);487 const auto start_tag = json_value(data, "reasoning_budget_start_tag", std::string());488 const auto end_tag = json_value(data, "reasoning_budget_end_tag", std::string());489 const auto message = json_value(data, "reasoning_budget_message", std::string());490 params.sampling.reasoning_budget_tokens = budget;491 492 if (!start_tag.empty()) {493 params.sampling.reasoning_budget_start = common_tokenize(vocab, start_tag, false, true);494 }495 if (!end_tag.empty()) {496 params.sampling.reasoning_budget_end = common_tokenize(vocab, end_tag, false, true);497 params.sampling.reasoning_budget_forced = common_tokenize(vocab, message + end_tag, false, true);498 499 SRV_DBG("reasoning budget: tokens=%d, generation_prompt='%s', start=%zu toks, end=%zu toks, forced=%zu toks\n",500 budget, params.sampling.generation_prompt.c_str(),501 params.sampling.reasoning_budget_start.size(),502 params.sampling.reasoning_budget_end.size(),503 params.sampling.reasoning_budget_forced.size());504 }505 }506 507 {508 params.sampling.logit_bias.clear();509 510 const auto & logit_bias = data.find("logit_bias");511 if (logit_bias != data.end() && logit_bias->is_array()) {512 const int n_vocab = llama_vocab_n_tokens(vocab);513 for (const auto & el : *logit_bias) {514 // TODO: we may want to throw errors here, in case "el" is incorrect515 if (el.is_array() && el.size() == 2) {516 float bias;517 if (el[1].is_number()) {518 bias = el[1].get<float>();519 } else if (el[1].is_boolean() && !el[1].get<bool>()) {520 bias = -INFINITY;521 } else {522 continue;523 }524 525 if (el[0].is_number_integer()) {526 llama_token tok = el[0].get<llama_token>();527 if (tok >= 0 && tok < n_vocab) {528 params.sampling.logit_bias.push_back({tok, bias});529 }530 } else if (el[0].is_string()) {531 auto toks = common_tokenize(vocab, el[0].get<std::string>(), false);532 for (auto tok : toks) {533 params.sampling.logit_bias.push_back({tok, bias});534 }535 }536 }537 }538 } else if (logit_bias != data.end() && logit_bias->is_object()) {539 const int n_vocab = llama_vocab_n_tokens(vocab);540 for (const auto & el : logit_bias->items()) {541 float bias;542 const auto & key = el.key();543 const auto & value = el.value();544 if (value.is_number()) {545 bias = value.get<float>();546 } else if (value.is_boolean() && !value.get<bool>()) {547 bias = -INFINITY;548 } else {549 continue;550 }551 552 char *end;553 llama_token tok = strtol(key.c_str(), &end, 10);554 if (*end == 0) {555 if (tok >= 0 && tok < n_vocab) {556 params.sampling.logit_bias.push_back({tok, bias});557 }558 } else {559 auto toks = common_tokenize(vocab, key, false);560 for (auto tok : toks) {561 params.sampling.logit_bias.push_back({tok, bias});562 }563 }564 }565 }566 567 params.sampling.ignore_eos = json_value(data, "ignore_eos", params_base.sampling.ignore_eos);568 if (params.sampling.ignore_eos) {569 params.sampling.logit_bias.insert(570 params.sampling.logit_bias.end(),571 logit_bias_eog.begin(), logit_bias_eog.end());572 }573 }574 575 {576 params.antiprompt.clear();577 578 const auto & stop = data.find("stop");579 if (stop != data.end() && stop->is_array()) {580 for (const auto & word : *stop) {581 if (!word.empty()) {582 params.antiprompt.push_back(word);583 }584 }585 }586 // set reverse prompt from cli args if not set in the request587 if (params.antiprompt.empty()) {588 params.antiprompt = defaults.antiprompt;589 }590 }591 592 {593 const auto samplers = data.find("samplers");594 if (samplers != data.end()) {595 if (samplers->is_array()) {596 params.sampling.samplers = common_sampler_types_from_names(*samplers, false);597 } else if (samplers->is_string()){598 params.sampling.samplers = common_sampler_types_from_chars(samplers->get<std::string>());599 }600 } else {601 params.sampling.samplers = defaults.sampling.samplers;602 }603 }604 605 if (params.n_cmpl > params_base.n_parallel) {606 throw std::runtime_error("n_cmpl cannot be greater than the number of slots, please increase -np");607 }608 609 return params;610}611 612//613// result_timings614//615 616json result_timings::to_json() const {617 json base = {618 {"cache_n", cache_n},619 620 {"prompt_n", prompt_n},621 {"prompt_ms", prompt_ms},622 {"prompt_per_token_ms", prompt_per_token_ms},623 {"prompt_per_second", prompt_per_second},624 625 {"predicted_n", predicted_n},626 {"predicted_ms", predicted_ms},627 {"predicted_per_token_ms", predicted_per_token_ms},628 {"predicted_per_second", predicted_per_second},629 };630 631 if (draft_n > 0) {632 base["draft_n"] = draft_n;633 base["draft_n_accepted"] = draft_n_accepted;634 }635 636 return base;637}638 639//640// result_prompt_progress641//642json result_prompt_progress::to_json() const {643 return json {644 {"total", total},645 {"cache", cache},646 {"processed", processed},647 {"time_ms", time_ms},648 };649}650 651static inline std::string stop_type_to_str(stop_type type) {652 switch (type) {653 case STOP_TYPE_EOS: return "eos";654 case STOP_TYPE_WORD: return "word";655 case STOP_TYPE_LIMIT: return "limit";656 default: return "none";657 }658}659 660//661// completion_token_output662//663 664json completion_token_output::to_json(bool post_sampling_probs) const {665 json probs_for_token = json::array();666 for (const auto & p : probs) {667 std::string txt(p.txt);668 txt.resize(validate_utf8(txt));669 probs_for_token.push_back(json {670 {"id", p.tok},671 {"token", txt},672 {"bytes", str_to_bytes(p.txt)},673 {674 post_sampling_probs ? "prob" : "logprob",675 post_sampling_probs ? p.prob : logarithm(p.prob)676 },677 });678 }679 return probs_for_token;680}681 682json completion_token_output::probs_vector_to_json(const std::vector<completion_token_output> & probs, bool post_sampling_probs) {683 json out = json::array();684 for (const auto & p : probs) {685 std::string txt(p.text_to_send);686 txt.resize(validate_utf8(txt));687 out.push_back(json {688 {"id", p.tok},689 {"token", txt},690 {"bytes", str_to_bytes(p.text_to_send)},691 {692 post_sampling_probs ? "prob" : "logprob",693 post_sampling_probs ? p.prob : logarithm(p.prob)694 },695 {696 post_sampling_probs ? "top_probs" : "top_logprobs",697 p.to_json(post_sampling_probs)698 },699 });700 }701 return out;702}703 704float completion_token_output::logarithm(float x) {705 // nlohmann::json converts -inf to null, so we need to prevent that706 return x == 0.0f ? std::numeric_limits<float>::lowest() : std::log(x);707}708 709std::vector<unsigned char> completion_token_output::str_to_bytes(const std::string & str) {710 std::vector<unsigned char> bytes;711 for (unsigned char c : str) {712 bytes.push_back(c);713 }714 return bytes;715}716 717//718// server_task_result_cmpl_final719//720json server_task_result_cmpl_final::to_json() {721 GGML_ASSERT(is_updated && "update() must be called before to_json()");722 switch (res_type) {723 case TASK_RESPONSE_TYPE_NONE:724 return to_json_non_oaicompat();725 case TASK_RESPONSE_TYPE_OAI_CMPL:726 return to_json_oaicompat();727 case TASK_RESPONSE_TYPE_OAI_CHAT:728 return stream ? to_json_oaicompat_chat_stream() : to_json_oaicompat_chat();729 case TASK_RESPONSE_TYPE_OAI_RESP:730 return stream ? to_json_oaicompat_resp_stream() : to_json_oaicompat_resp();731 case TASK_RESPONSE_TYPE_OAI_ASR:732 return to_json_oaicompat_asr();733 case TASK_RESPONSE_TYPE_ANTHROPIC:734 return stream ? to_json_anthropic_stream() : to_json_anthropic();735 default:736 GGML_ASSERT(false && "Invalid task_response_type");737 }738}739 740json server_task_result_cmpl_final::to_json_non_oaicompat() {741 json res = json {742 {"index", index},743 {"content", content},744 {"tokens", tokens},745 {"id_slot", id_slot},746 {"stop", true},747 {"model", oaicompat_model},748 {"tokens_predicted", n_decoded},749 {"tokens_evaluated", n_prompt_tokens},750 {"generation_settings", generation_params.to_json()},751 {"prompt", prompt},752 {"has_new_line", has_new_line},753 {"truncated", truncated},754 {"stop_type", stop_type_to_str(stop)},755 {"stopping_word", stopping_word},756 {"tokens_cached", n_tokens_cached},757 {"timings", timings.to_json()},758 };759 if (!stream && !probs_output.empty()) {760 res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs);761 }762 return response_fields.empty() ? res : json_get_nested_values(response_fields, res);763}764 765json server_task_result_cmpl_final::usage_json_oaicompat() {766 return json {767 {"completion_tokens", n_decoded},768 {"prompt_tokens", n_prompt_tokens},769 {"total_tokens", n_decoded + n_prompt_tokens},770 {"prompt_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},771 };772}773 774json server_task_result_cmpl_final::to_json_oaicompat() {775 std::time_t t = std::time(0);776 json logprobs = json(nullptr); // OAI default to null777 if (!stream && probs_output.size() > 0) {778 logprobs = json{779 {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},780 };781 }782 json finish_reason = "length";783 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {784 finish_reason = "stop";785 }786 json res = json {787 {"choices", json::array({788 json{789 {"text", content},790 {"index", index},791 {"logprobs", logprobs},792 {"finish_reason", finish_reason},793 }794 })},795 {"created", t},796 {"model", oaicompat_model},797 {"system_fingerprint", std::string(llama_build_info())},798 {"object", "text_completion"},799 {"usage", usage_json_oaicompat()},800 {"id", oaicompat_cmpl_id}801 };802 803 // extra fields for debugging purposes804 if (verbose) {805 res["__verbose"] = to_json_non_oaicompat();806 }807 if (timings.prompt_n >= 0) {808 res.push_back({"timings", timings.to_json()});809 }810 811 return res;812}813 814json server_task_result_cmpl_final::to_json_oaicompat_chat() {815 std::string finish_reason = "length";816 common_chat_msg msg;817 if (!oaicompat_msg.empty()) {818 msg = oaicompat_msg;819 } else {820 msg.role = "assistant";821 msg.content = content;822 }823 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {824 finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";825 }826 827 json choice {828 {"finish_reason", finish_reason},829 {"index", index},830 {"message", msg.to_json_oaicompat()},831 };832 833 if (!stream && probs_output.size() > 0) {834 choice["logprobs"] = json{835 {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)},836 };837 }838 839 std::time_t t = std::time(0);840 841 json res = json {842 {"choices", json::array({choice})},843 {"created", t},844 {"model", oaicompat_model},845 {"system_fingerprint", std::string(llama_build_info())},846 {"object", "chat.completion"},847 {"usage", usage_json_oaicompat()},848 {"id", oaicompat_cmpl_id}849 };850 851 // extra fields for debugging purposes852 if (verbose) {853 res["__verbose"] = to_json_non_oaicompat();854 }855 if (timings.prompt_n >= 0) {856 res.push_back({"timings", timings.to_json()});857 }858 859 return res;860}861 862json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() {863 std::time_t t = std::time(0);864 std::string finish_reason = "length";865 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {866 finish_reason = oaicompat_msg.tool_calls.empty() ? "stop" : "tool_calls";867 }868 869 json deltas = json::array();870 for (const auto & diff : oaicompat_msg_diffs) {871 deltas.push_back({872 {"choices", json::array({873 json {874 {"finish_reason", nullptr},875 {"index", index},876 {"delta", common_chat_msg_diff_to_json_oaicompat(diff)},877 },878 })},879 {"created", t},880 {"id", oaicompat_cmpl_id},881 {"model", oaicompat_model},882 {"system_fingerprint", std::string(llama_build_info())},883 {"object", "chat.completion.chunk"},884 });885 }886 887 deltas.push_back({888 {"choices", json::array({889 json {890 {"finish_reason", finish_reason},891 {"index", index},892 {"delta", json::object()},893 },894 })},895 {"created", t},896 {"id", oaicompat_cmpl_id},897 {"model", oaicompat_model},898 {"system_fingerprint", std::string(llama_build_info())},899 {"object", "chat.completion.chunk"},900 });901 902 if (include_usage) {903 // OpenAI API spec for chat.completion.chunks specifies an empty `choices` array for the last chunk when including usage904 // https://platform.openai.com/docs/api-reference/chat_streaming/streaming#chat_streaming/streaming-choices905 deltas.push_back({906 {"choices", json::array()},907 {"created", t},908 {"id", oaicompat_cmpl_id},909 {"model", oaicompat_model},910 {"system_fingerprint", std::string(llama_build_info())},911 {"object", "chat.completion.chunk"},912 {"usage", usage_json_oaicompat()},913 });914 }915 916 if (timings.prompt_n >= 0) {917 deltas.back().push_back({"timings", timings.to_json()});918 }919 920 // extra fields for debugging purposes921 if (verbose && !deltas.empty()) {922 deltas.front()["__verbose"] = to_json_non_oaicompat();923 }924 925 return deltas;926}927 928json server_task_result_cmpl_final::to_json_oaicompat_resp() {929 common_chat_msg msg;930 if (!oaicompat_msg.empty()) {931 msg = oaicompat_msg;932 } else {933 msg.role = "assistant";934 msg.content = content;935 }936 937 std::vector<json> output;938 939 if (msg.reasoning_content != "") {940 output.push_back(json {941 {"id", "rs_" + random_string()},942 {"summary", json::array()},943 {"type", "reasoning"},944 {"content", json::array({ json {945 {"text", msg.reasoning_content},946 {"type", "reasoning_text"},947 }})},948 {"encrypted_content", ""},949 {"status", "completed"},950 });951 }952 953 if (msg.content != "") {954 output.push_back(json {955 {"content", json::array({ json {956 {"type", "output_text"},957 {"annotations", json::array()},958 {"logprobs", json::array()},959 {"text", msg.content},960 }})},961 {"id", "msg_" + random_string()},962 {"role", msg.role},963 {"status", "completed"},964 {"type", "message"},965 });966 }967 968 for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) {969 output.push_back(json {970 {"type", "function_call"},971 {"status", "completed"},972 {"arguments", tool_call.arguments},973 {"call_id", "fc_" + tool_call.id},974 {"name", tool_call.name},975 });976 }977 978 std::time_t t = std::time(0);979 json res = {980 {"completed_at", t},981 {"created_at", t},982 {"id", oai_resp_id},983 {"model", oaicompat_model},984 {"object", "response"},985 {"output", output},986 {"status", "completed"},987 {"usage", json {988 {"input_tokens", n_prompt_tokens},989 {"output_tokens", n_decoded},990 {"total_tokens", n_decoded + n_prompt_tokens},991 {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},992 }},993 };994 995 return res;996}997 998json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {999 std::vector<json> server_sent_events;1000 std::vector<json> output;1001 1002 if (oaicompat_msg.reasoning_content != "") {1003 const json output_item = json {1004 {"id", oai_resp_reasoning_id},1005 {"summary", json::array()},1006 {"type", "reasoning"},1007 {"content", json::array({ json {1008 {"text", oaicompat_msg.reasoning_content},1009 {"type", "reasoning_text"},1010 }})},1011 {"encrypted_content", ""},1012 };1013 1014 server_sent_events.push_back(json {1015 {"event", "response.output_item.done"},1016 {"data", json {1017 {"type", "response.output_item.done"},1018 {"item", output_item}1019 }}1020 });1021 output.push_back(output_item);1022 }1023 1024 if (oaicompat_msg.content != "") {1025 server_sent_events.push_back(json {1026 {"event", "response.output_text.done"},1027 {"data", json {1028 {"type", "response.output_text.done"},1029 {"item_id", oai_resp_message_id},1030 {"text", oaicompat_msg.content}1031 }}1032 });1033 1034 const json content_part = {1035 {"type", "output_text"},1036 {"annotations", json::array()},1037 {"logprobs", json::array()},1038 {"text", oaicompat_msg.content}1039 };1040 1041 server_sent_events.push_back(json {1042 {"event", "response.content_part.done"},1043 {"data", json {1044 {"type", "response.content_part.done"},1045 {"item_id", oai_resp_message_id},1046 {"part", content_part}1047 }}1048 });1049 const json output_item = {1050 {"type", "message"},1051 {"status", "completed"},1052 {"id", oai_resp_message_id},1053 {"content", json::array({content_part})},1054 {"role", "assistant"}1055 };1056 1057 server_sent_events.push_back(json {1058 {"event", "response.output_item.done"},1059 {"data", json {1060 {"type", "response.output_item.done"},1061 {"item", output_item}1062 }}1063 });1064 output.push_back(output_item);1065 }1066 1067 for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) {1068 const json output_item = {1069 {"type", "function_call"},1070 {"status", "completed"},1071 {"arguments", tool_call.arguments},1072 {"call_id", "fc_" + tool_call.id},1073 {"name", tool_call.name}1074 };1075 server_sent_events.push_back(json {1076 {"event", "response.output_item.done"},1077 {"data", json {1078 {"type", "response.output_item.done"},1079 {"item", output_item}1080 }}1081 });1082 output.push_back(output_item);1083 }1084 1085 std::time_t t = std::time(0);1086 server_sent_events.push_back(json {1087 {"event", "response.completed"},1088 {"data", json {1089 {"type", "response.completed"},1090 {"response", json {1091 {"id", oai_resp_id},1092 {"object", "response"},1093 {"created_at", t},1094 {"status", "completed"},1095 {"model", oaicompat_model},1096 {"output", output},1097 {"usage", json {1098 {"input_tokens", n_prompt_tokens},1099 {"output_tokens", n_decoded},1100 {"total_tokens", n_decoded + n_prompt_tokens},1101 {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},1102 }}1103 }},1104 }}1105 });1106 1107 return server_sent_events;1108}1109 1110json server_task_result_cmpl_final::to_json_oaicompat_asr() {1111 json event = json {1112 {"type", "transcript.text.done"},1113 {"text", content},1114 {"usage", json {1115 {"type", "tokens"},1116 {"input_tokens", n_prompt_tokens},1117 {"output_tokens", n_decoded},1118 {"total_tokens", n_decoded + n_prompt_tokens},1119 {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }},1120 }},1121 };1122 return event;1123}1124 1125json server_task_result_cmpl_final::to_json_anthropic() {1126 std::string stop_reason = "max_tokens";1127 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {1128 stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use";1129 }1130 1131 json content_blocks = json::array();1132 1133 common_chat_msg msg;1134 if (!oaicompat_msg.empty()) {1135 msg = oaicompat_msg;1136 } else {1137 msg.role = "assistant";1138 msg.content = content;1139 }1140 1141 // thinking block comes first (Anthropic extended thinking format)1142 if (!msg.reasoning_content.empty()) {1143 content_blocks.push_back({1144 {"type", "thinking"},1145 {"thinking", msg.reasoning_content},1146 {"signature", ""} // empty signature for local models (no cryptographic verification)1147 });1148 }1149 1150 if (!msg.content.empty()) {1151 content_blocks.push_back({1152 {"type", "text"},1153 {"text", msg.content}1154 });1155 }1156 1157 for (const auto & tool_call : msg.tool_calls) {1158 json tool_use_block = {1159 {"type", "tool_use"},1160 {"id", tool_call.id},1161 {"name", tool_call.name}1162 };1163 1164 try {1165 tool_use_block["input"] = json::parse(tool_call.arguments);1166 } catch (const std::exception &) {1167 tool_use_block["input"] = json::object();1168 }1169 1170 content_blocks.push_back(tool_use_block);1171 }1172 1173 json res = {1174 {"id", oaicompat_cmpl_id},1175 {"type", "message"},1176 {"role", "assistant"},1177 {"content", content_blocks},1178 {"model", oaicompat_model},1179 {"stop_reason", stop_reason},1180 {"stop_sequence", stopping_word.empty() ? nullptr : json(stopping_word)},1181 {"usage", {1182 {"cache_read_input_tokens", n_prompt_tokens_cache},1183 {"input_tokens", n_prompt_tokens - n_prompt_tokens_cache},1184 {"output_tokens", n_decoded}1185 }}1186 };1187 1188 return res;1189}1190 1191json server_task_result_cmpl_final::to_json_anthropic_stream() {1192 json events = json::array();1193 1194 std::string stop_reason = "max_tokens";1195 if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {1196 stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use";1197 }1198 1199 bool has_thinking = !oaicompat_msg.reasoning_content.empty();1200 bool has_text = !oaicompat_msg.content.empty();