Felipe97/llama-cpp-compiled

1#include "server-schema.h"2 3#include "json-schema-to-grammar.h"4 5namespace server_schema {6 7//8// llama.cpp-specific completion schema9//10 11std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params & params_base, task_params & params) {12    std::vector<std::unique_ptr<field>> fields;13    auto add = [&](field * f) {14        fields.emplace_back(f);15    };16 17    add((new field_bool("verbose", params.verbose))18        ->set_desc("Include __verbose field in the response with additional debug information"));19 20    add((new field_bool("timings_per_token", params.timings_per_token))21        ->set_desc("Include prompt processing and text generation speed information in each response"));22 23    add((new field_bool("stream", params.stream))24        ->set_desc("Allows receiving each predicted token in real-time instead of waiting for the completion to finish"));25 26    add((new field_nested("stream_options"))27        ->add_subfield((new field_bool("include_usage", params.include_usage))28            ->set_desc("Whether to include usage information in the stream"))29        ->set_desc("Additional options for streaming responses"));30 31    add((new field_bool("cache_prompt", params.cache_prompt))32        ->set_desc("Re-use KV cache from a previous request if possible. This way the common prefix does not have to be re-processed, only the suffix that differs between the requests"));33 34    add((new field_bool("return_tokens", params.return_tokens))35        ->set_desc("Return the raw generated token ids in the `tokens` field"));36 37    add((new field_bool("return_progress", params.return_progress))38        ->set_desc("Include prompt processing progress events in stream mode"));39 40    add((new field_num("sse_ping_interval", params.sse_ping_interval))41        ->set_hard_limits(-1, INT32_MAX)42        ->set_desc("Interval in seconds between SSE comment pings emitted while the stream stays silent, -1 disables pings"));43 44    add((new field_num("n_predict", params.n_predict))45        ->set_hard_limits(-1, INT32_MAX)46        ->add_alias("max_completion_tokens")47        ->add_alias("max_tokens")48        ->set_desc("Set the maximum number of tokens to predict. When 0, no tokens will be generated but the prompt is evaluated into the cache"));49 50    add((new field_num("n_indent", params.n_indent))51        ->set_hard_limits(0, INT32_MAX)52        ->set_desc("Specify the minimum line indentation for the generated text in number of whitespace characters. Useful for code completion tasks"));53 54    add((new field_num("n_keep", params.n_keep))55        ->set_hard_limits(-1, INT32_MAX)56        ->set_desc("Specify the number of tokens from the initial prompt to retain when context size is exceeded. Use -1 to retain all tokens from the prompt"));57 58    add((new field_num("n_discard", params.n_discard))59        ->set_hard_limits(0, INT32_MAX)60        ->set_desc("Number of tokens after n_keep that may be discarded when shifting context (0 = half context)"));61 62    add((new field_num("n_cmpl", params.n_cmpl))63        ->set_hard_limits(1, params_base.n_parallel)64        ->add_alias("n") // alias "n" as fallback (OpenAI completions API)65        ->set_desc("Number of completions to generate. If the input has multiple prompts, total outputs will be N prompts times n_cmpl"));66 67    add((new field_num("n_cache_reuse", params.n_cache_reuse))68        ->set_hard_limits(0, INT32_MAX)69        ->set_desc("Min chunk size to attempt reusing from the cache via KV shifting. See --cache-reuse arg"));70 71    // TODO: implement t_max_prompt_ms72    // add((new field_num("t_max_prompt_ms", params.t_max_prompt_ms))73 74    add((new field_num("t_max_predict_ms", params.t_max_predict_ms))75        ->set_hard_limits(-1, std::numeric_limits<int64_t>::max())76        ->set_desc("Set a time limit in milliseconds for the prediction phase. The timeout triggers if generation exceeds this time (measured since the first token) and a newline has been generated. Useful for FIM applications"));77 78    add((new field_json("response_fields"))79        ->set_desc("A list of response fields to return. Missing fields are omitted without error. Fields with a slash are unnested (e.g. generation_settings/n_predict moves n_predict to the root)")80        ->set_handler([&](field_eval_context & ctx, const json & data) {81            ctx.params.response_fields = json_value(data, "response_fields", std::vector<std::string>());82        }));83 84 85    //86    // Sampling params87    //88 89    add((new field_num("top_k", params.sampling.top_k))90        ->set_limits(0, INT32_MAX)91        ->set_desc("Limit the next token selection to the K most probable tokens (0 = disabled)"));92 93    add((new field_num("top_p", params.sampling.top_p))94        ->set_limits(0.0f, 1.0f)95        ->set_desc("Limit the next token selection to a subset of tokens with cumulative probability above threshold P (1.0 = disabled)"));96 97    add((new field_num("min_p", params.sampling.min_p))98        ->set_limits(0.0f, 1.0f)99        ->set_desc("The minimum probability for a token to be considered, relative to the probability of the most likely token (0 = disabled)"));100 101    add((new field_num("top_n_sigma", params.sampling.top_n_sigma))102        ->set_desc("Keep tokens within n standard deviations of the top token logit (< 0 = disabled)"));103 104    add((new field_num("xtc_probability", params.sampling.xtc_probability))105        ->set_limits(0.0f, 1.0f)106        ->set_desc("Set the chance for token removal via XTC sampler (0 = disabled)"));107 108    add((new field_num("xtc_threshold", params.sampling.xtc_threshold))109        ->set_limits(0.0f, 1.0f)110        ->set_desc("Set a minimum probability threshold for tokens to be removed via XTC sampler (> 0.5 disables XTC)"));111 112    add((new field_num("typical_p", params.sampling.typ_p))113        // ->set_limits(0.0f, 1.0f) // what's the valid range?114        ->set_desc("Enable locally typical sampling with parameter p (1.0 = disabled)"));115 116    add((new field_num("temperature", params.sampling.temp))117        ->set_limits(0.0f, std::numeric_limits<float>::infinity())118        ->set_desc("Adjust the randomness of the generated text (0 = greedy)"));119 120    add((new field_num("dynatemp_range", params.sampling.dynatemp_range))121        ->set_desc("Dynamic temperature range. The final temperature will be in [temperature - range, temperature + range] (0 = disabled)"));122 123    add((new field_num("dynatemp_exponent", params.sampling.dynatemp_exponent))124        ->set_desc("Dynamic temperature exponent, controls how entropy maps to temperature"));125 126    add((new field_num("repeat_last_n", params.sampling.penalty_last_n))127        ->set_hard_limits(0, INT32_MAX)128        ->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled)"));129 130    add((new field_num("repeat_penalty", params.sampling.penalty_repeat))131        ->set_desc("Control the repetition of token sequences in the generated text (1.0 = disabled)"));132 133    add((new field_num("frequency_penalty", params.sampling.penalty_freq))134        ->set_desc("Repeat alpha frequency penalty (0 = disabled)"));135 136    add((new field_num("presence_penalty", params.sampling.penalty_present))137        ->set_desc("Repeat alpha presence penalty (0 = disabled)"));138 139    add((new field_num("dry_multiplier", params.sampling.dry_multiplier))140        ->set_desc("Set the DRY (Don't Repeat Yourself) repetition penalty multiplier (0 = disabled)"));141 142    add((new field_num("dry_base", params.sampling.dry_base))143        ->set_desc("Set the DRY repetition penalty base value (must be >= 1.0, any values < 1.0 will be replaced with the default value)")144        ->set_handler([&](field_eval_context & ctx, const json & data) {145            float v = data.at("dry_base").get<float>();146            ctx.params.sampling.dry_base = (v < 1.0f) ? params_base.sampling.dry_base : v;147        }));148 149    add((new field_num("dry_allowed_length", params.sampling.dry_allowed_length))150        ->set_hard_limits(0, INT32_MAX)151        ->set_desc("Tokens that extend repetition beyond this length receive exponentially increasing penalty: multiplier * base ^ (sequence_length - allowed_length)"));152 153    add((new field_num("dry_penalty_last_n", params.sampling.dry_penalty_last_n))154        ->set_hard_limits(0, INT32_MAX)155        ->set_desc("How many tokens to scan for repetitions (0 = disabled)"));156 157    add((new field_num("mirostat", params.sampling.mirostat))158        ->set_limits(0, 2)159        ->set_desc("Enable Mirostat sampling, controlling perplexity during text generation (0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)"));160 161    add((new field_num("mirostat_tau", params.sampling.mirostat_tau))162        ->set_desc("Set the Mirostat target entropy, parameter tau"));163 164    add((new field_num("mirostat_eta", params.sampling.mirostat_eta))165        ->set_desc("Set the Mirostat learning rate, parameter eta"));166 167    add((new field_num("adaptive_target", params.sampling.adaptive_target))168        ->set_limits(-std::numeric_limits<float>::max(), 1.0f)169        ->set_desc("Adaptive sampling target entropy (valid range 0.0 to 1.0; negative = disabled)"));170 171    add((new field_num("adaptive_decay", params.sampling.adaptive_decay))172        ->set_hard_limits(0.0f, 0.99f)173        ->set_desc("EMA decay for adaptive sampling; history approximates 1/(1-decay) tokens"));174 175    // seed is uint32_t; field_num uses int32_t so use a handler176    add((new field_num("seed", params.sampling.seed))177        ->set_desc("Set the random number generator (RNG) seed (-1 = random)"));178 179    add((new field_num("n_probs", params.sampling.n_probs))180        ->add_alias("logprobs") // use "logprobs" if "n_probs" wasn't provided181        ->set_desc("If greater than 0, output the probabilities of top N tokens for each generated token"));182 183    add((new field_num("min_keep", params.sampling.min_keep))184        ->set_hard_limits(0, INT32_MAX)185        ->set_desc("If greater than 0, force samplers to return at least N possible tokens"));186 187    add((new field_bool("backend_sampling", params.sampling.backend_sampling))188        ->set_desc("Use backend sampling instead of llama.cpp sampling"));189 190    add((new field_bool("post_sampling_probs", params.post_sampling_probs))191        ->set_desc("Return probabilities of top n_probs tokens after applying the sampling chain"));192 193    //194    // Speculative decoding params195    //196 197    // TODO: to keep things simple, we disable speculative parameter adjustments for now198#if 0199    // TODO: for now, be able to adjust only the draft-model based speculative parameters200    add((new field_num("speculative.n_max", params.speculative.draft.n_max))201        ->set_hard_limits(0, INT32_MAX)202        ->set_desc("Maximum number of tokens to draft during speculative decoding"));203 204    add((new field_num("speculative.n_min", params.speculative.draft.n_min))205        ->set_hard_limits(0, INT32_MAX)206        ->set_desc("Minimum number of draft tokens to use for speculative decoding");207 208    add((new field_num("speculative.p_min", params.speculative.draft.p_min))209        ->set_hard_limits(0.0f, 1.0f)210        ->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)"));211 212 213    add((new field_str("speculative.type"))214        ->set_desc("Speculative decoding method (for debugging and research purposes)")215        ->set_handler([&](field_eval_context & ctx, const json & data) {216            ctx.params.speculative.types = { common_speculative_type_from_name(data.at("speculative.type").get<std::string>()) };217        }));218 219    add((new field_num("speculative.ngram_size_n", params.speculative.ngram_simple.size_n))220        ->set_desc("Ngram size for lookup in ngram-based speculative decoding"));221 222    add((new field_num("speculative.ngram_size_m", params.speculative.ngram_simple.size_m))223        ->set_desc("Mgram size for speculative tokens in ngram-based speculative decoding"));224 225    add((new field_num("speculative.ngram_min_hits", params.speculative.ngram_simple.min_hits))226        ->set_desc("Minimum hits at ngram lookup for mgram to be proposed"));227#endif228 229    add((new field_json("lora"))230        ->set_desc("A list of LoRA adapters to apply to this request. Each entry must have `id` and `scale` fields. Adapters not listed default to scale 0.0")231        ->set_handler([&](field_eval_context & ctx, const json & data) {232            const auto & lora = data.at("lora");233            if (!lora.is_array()) {234                throw std::runtime_error("Error: 'lora' must be an array of objects with 'id' and 'scale' fields");235            }236            ctx.params.lora = parse_lora_request(lora);237        }));238 239    // sequence breakers for DRY240    // Currently, this is not compatible with TextGen WebUI, Koboldcpp and SillyTavern format241    // Ref: https://github.com/oobabooga/text-generation-webui/blob/d1af7a41ade7bd3c3a463bfa640725edb818ebaf/extensions/openai/typing.py#L39242    add((new field_json("dry_sequence_breakers"))243        ->set_desc("Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted")244        ->set_handler([&](field_eval_context & ctx, const json & data) {245            ctx.params.sampling.dry_sequence_breakers = json_value(data, "dry_sequence_breakers", std::vector<std::string>());246            if (ctx.params.sampling.dry_sequence_breakers.empty()) {247                throw std::runtime_error("Error: dry_sequence_breakers must be a non-empty array of strings");248            }249        }));250 251    // handle both "json_schema" and "grammar"252    add((new field_json("json_schema"))253        ->add_alias("grammar")254        ->set_desc("Set a JSON schema (json_schema) or GBNF grammar string (grammar) for constrained generation. json_schema takes precedence if both are provided")255        ->set_handler([&](field_eval_context & ctx, const json & data) {256            auto & params = ctx.params;257            if (data.contains("json_schema") && !data.contains("grammar")) {258                try {259                    auto schema                  = json_value(data, "json_schema", json::object());260                    if (schema.is_object() && schema.empty()) {261                        // an empty schema means any object262                        schema["type"] = "object";263                    }264                    SRV_DBG("JSON schema: %s\n", schema.dump(2).c_str());265                    std::string grammar_str      = json_schema_to_grammar(schema);266                    SRV_DBG("Converted grammar: %s\n", grammar_str.c_str());267                    params.sampling.grammar      = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, std::move(grammar_str)};268                } catch (const std::exception & e) {269                    throw std::runtime_error(std::string("\"json_schema\": ") + e.what());270                }271            } else {272                std::string grammar_str = json_value(data, "grammar", std::string());273                if (!grammar_str.empty()) {274                    // grammar_type key is set by the server when converting chat template grammars275                    std::string grammar_type = json_value(data, "grammar_type", std::string());276                    if (grammar_type == "tool_calls") {277                        params.sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, std::move(grammar_str)};278                    } else {279                        // explicit grammar from the user (API field "grammar")280                        params.sampling.grammar = {COMMON_GRAMMAR_TYPE_USER, std::move(grammar_str)};281                    }282                    SRV_DBG("Grammar (%s): %s\n", grammar_type.c_str(), common_grammar_value(params.sampling.grammar).c_str());283                }284            }285        }));286 287    add((new field_bool("grammar_lazy", params.sampling.grammar_lazy))288        ->set_desc("Whether to apply grammar constraints lazily, only when triggered (instead of at every step)"));289 290    //291    // Chat parser params292    //293 294    // TODO: change this to string field instead295    add((new field_json("chat_format"))296        ->set_desc("Chat format used internally by the server")297        ->set_handler([&](field_eval_context & ctx, const json & data) {298            ctx.params.chat_parser_params.format = static_cast<common_chat_format>(data.at("chat_format").get<int>());299            SRV_TRC("chat format: %s\n", common_chat_format_name(ctx.params.chat_parser_params.format));300        }));301 302    add((new field_str("reasoning_format"))303        ->set_desc("Reasoning format for chain-of-thought models")304        ->set_handler([&](field_eval_context & ctx, const json & data) {305            auto reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());306            ctx.params.chat_parser_params.reasoning_format = reasoning_format;307            ctx.params.chat_parser_params.reasoning_in_content = ctx.params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);308        }));309 310    add((new field_str("generation_prompt"))311        ->set_desc("Generation prompt appended to the chat template output")312        ->set_handler([&](field_eval_context & ctx, const json & data) {313            std::string s = data.at("generation_prompt").get<std::string>();314            ctx.params.chat_parser_params.generation_prompt = s;315            ctx.params.sampling.generation_prompt = s;316        }));317 318    add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))319        ->set_desc("Whether to parse tool calls from the generated output"));320 321    add((new field_str("chat_parser"))322        ->set_desc("Chat parser configuration string")323        ->set_handler([&](field_eval_context & ctx, const json & data) {324            ctx.params.chat_parser_params.parser.load(data.at("chat_parser").get<std::string>());325        }));326 327    add((new field_json("continue_final_message"))328        ->set_desc("Whether to continue the final message of the chat template")329        ->set_handler([&](field_eval_context & ctx, const json & data) {330            auto continuation = common_chat_continuation_parse(data.at("continue_final_message"));331            ctx.params.chat_parser_params.is_continuation = continuation != COMMON_CHAT_CONTINUATION_NONE;332        }));333 334    add((new field_bool("echo", params.chat_parser_params.echo))335        ->set_desc("Whether to echo the input tokens in the output"));336 337    //338    // Token-level fields (require vocab)339    //340 341    add((new field_json("preserved_tokens"))342        ->set_desc("List of token strings that must not be split during tokenization")343        ->set_handler([&](field_eval_context & ctx, const json & data) {344            GGML_ASSERT(ctx.vocab != nullptr);345            for (const auto & t : data.at("preserved_tokens")) {346                auto ids = common_tokenize(ctx.vocab, t.get<std::string>(), false, true);347                if (ids.size() == 1) {348                    ctx.params.sampling.preserved_tokens.insert(ids[0]);349                }350            }351        }));352 353    add((new field_json("grammar_triggers"))354        ->set_desc("List of strings or patterns that trigger grammar-constrained generation")355        ->set_handler([&](field_eval_context & ctx, const json & data) {356            GGML_ASSERT(ctx.vocab != nullptr);357            for (const auto & t : data.at("grammar_triggers")) {358                server_grammar_trigger ct(t);359                if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {360                    const auto & word = ct.value.value;361                    auto ids = common_tokenize(ctx.vocab, word, false, true);362                    if (ids.size() == 1) {363                        auto token = ids[0];364                        if (std::find(ctx.params.sampling.preserved_tokens.begin(), ctx.params.sampling.preserved_tokens.end(), (llama_token) token) == ctx.params.sampling.preserved_tokens.end()) {365                            throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);366                        }367                        common_grammar_trigger trigger;368                        trigger.type  = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;369                        trigger.value = word;370                        trigger.token = token;371                        ctx.params.sampling.grammar_triggers.push_back(std::move(trigger));372                    } else {373                        ctx.params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});374                    }375                } else {376                    ctx.params.sampling.grammar_triggers.emplace_back(std::move(ct.value));377                }378            }379            if (ctx.params.sampling.grammar_lazy && ctx.params.sampling.grammar_triggers.empty()) {380                throw std::runtime_error("Error: no triggers set for lazy grammar!");381            }382        }));383 384    add((new field_bool("reasoning_control", params.sampling.reasoning_control))385        ->set_desc("Create the budget sampler on demand so reasoning can be ended at runtime"));386 387    add((new field_num("reasoning_budget_tokens", params.sampling.reasoning_budget_tokens))388        ->set_hard_limits(-1, INT32_MAX)389        ->set_desc("Number of tokens in the reasoning budget (-1 = disabled)"));390 391    add((new field_str("reasoning_budget_start_tag"))392        ->set_desc("Token string marking the start of the reasoning budget section")393        ->set_handler([&](field_eval_context & ctx, const json & data) {394            GGML_ASSERT(ctx.vocab != nullptr);395            ctx.params.sampling.reasoning_budget_start = common_tokenize(ctx.vocab, data.at("reasoning_budget_start_tag").get<std::string>(), false, true);396        }));397 398    add((new field_json("reasoning_budget_end_tags"))399        ->add_alias("reasoning_budget_end_tag")400        ->set_desc("Token strings marking the end of the reasoning budget section; the first is forced when the budget expires")401        ->set_handler([&](field_eval_context & ctx, const json & data) {402            GGML_ASSERT(ctx.vocab != nullptr);403            ctx.params.sampling.reasoning_budget_end.clear();404            if (data.contains("reasoning_budget_end_tags")) {405                for (const auto & t : data.at("reasoning_budget_end_tags")) {406                    std::string tag = t.get<std::string>();407                    if (!tag.empty()) {408                        ctx.params.sampling.reasoning_budget_end.push_back(common_tokenize(ctx.vocab, tag, false, true));409                    }410                }411            } else if (data.contains("reasoning_budget_end_tag")) {412                std::string tag = data.at("reasoning_budget_end_tag").get<std::string>();413                if (!tag.empty()) {414                    ctx.params.sampling.reasoning_budget_end.push_back(common_tokenize(ctx.vocab, tag, false, true));415                }416            }417        }));418 419    add((new field_str("reasoning_budget_message"))420        ->set_desc("Message to prepend to the reasoning budget end tag when forcing it")421        ->set_handler([&](field_eval_context & ctx, const json & data) {422            GGML_ASSERT(ctx.vocab != nullptr);423            if (!ctx.params.sampling.reasoning_budget_end.empty()) {424                llama_tokens end_tag = ctx.params.sampling.reasoning_budget_end.front();425                std::string message = json_value(data, "reasoning_budget_message", std::string());426                if (!message.empty()) {427                    llama_tokens message_tokens = common_tokenize(ctx.vocab, message, false, true);428                    end_tag.insert(end_tag.begin(), message_tokens.begin(), message_tokens.end());429                }430                ctx.params.sampling.reasoning_budget_forced = std::move(end_tag);431            }432        }));433 434    add((new field_json("logit_bias"))435        ->set_desc("Modify the likelihood of specific tokens. Accepts an array of [token, bias] pairs or an object mapping token to bias. Use false as bias to ban a token")436        ->set_handler([&](field_eval_context & ctx, const json & data) {437            GGML_ASSERT(ctx.vocab != nullptr);438            ctx.params.sampling.logit_bias.clear();439            const auto & logit_bias = data.at("logit_bias");440            const int n_vocab = llama_vocab_n_tokens(ctx.vocab);441            auto parse_bias = [](const json & v, float & bias) -> bool {442                if (v.is_number())                        { bias = v.get<float>(); return true; }443                if (v.is_boolean() && !v.get<bool>())     { bias = -INFINITY;      return true; }444                return false;445            };446            if (logit_bias.is_array()) {447                for (const auto & el : logit_bias) {448                    if (!el.is_array() || el.size() != 2) continue;449                    float bias;450                    if (!parse_bias(el[1], bias)) continue;451                    if (el[0].is_number_integer()) {452                        llama_token tok = el[0].get<llama_token>();453                        if (tok >= 0 && tok < n_vocab) ctx.params.sampling.logit_bias.push_back({tok, bias});454                    } else if (el[0].is_string()) {455                        for (auto tok : common_tokenize(ctx.vocab, el[0].get<std::string>(), false))456                            ctx.params.sampling.logit_bias.push_back({tok, bias});457                    }458                }459            } else if (logit_bias.is_object()) {460                for (const auto & el : logit_bias.items()) {461                    float bias;462                    if (!parse_bias(el.value(), bias)) continue;463                    char * end;464                    llama_token tok = strtol(el.key().c_str(), &end, 10);465                    if (*end == 0) {466                        if (tok >= 0 && tok < n_vocab) ctx.params.sampling.logit_bias.push_back({tok, bias});467                    } else {468                        for (auto t : common_tokenize(ctx.vocab, el.key(), false))469                            ctx.params.sampling.logit_bias.push_back({t, bias});470                    }471                }472            }473        }));474 475    add((new field_bool("ignore_eos", params.sampling.ignore_eos))476        ->set_desc("Ignore the end-of-sequence token and continue generating")477        ->set_handler([&](field_eval_context & ctx, const json & data) {478            GGML_ASSERT(ctx.logit_bias_eog != nullptr);479            ctx.params.sampling.ignore_eos = data.at("ignore_eos").get<bool>();480            if (ctx.params.sampling.ignore_eos && ctx.logit_bias_eog) {481                ctx.params.sampling.logit_bias.insert(482                    ctx.params.sampling.logit_bias.end(),483                    ctx.logit_bias_eog->begin(), ctx.logit_bias_eog->end());484            }485        }));486 487    add((new field_json("stop"))488        ->set_desc("Specify stopping strings. Generation stops when one is produced, and the string is not included in the output")489        ->set_handler([&](field_eval_context & ctx, const json & data) {490            ctx.params.antiprompt.clear();491            const auto & stop = data.at("stop");492            if (stop.is_array()) {493                for (const auto & word : stop) {494                    if (!word.empty()) ctx.params.antiprompt.push_back(word);495                }496            } else if (stop.is_string()) {497                ctx.params.antiprompt.push_back(stop.get<std::string>());498            }499            // fall back to CLI defaults if the request provided no effective stop strings500            if (ctx.params.antiprompt.empty()) {501                ctx.params.antiprompt = params_base.antiprompt;502            }503        }));504 505    add((new field_json("samplers"))506        ->set_desc("The order in which samplers are applied. An array of sampler type names, or a single string of sampler chars")507        ->set_handler([&](field_eval_context & ctx, const json & data) {508            const auto & samplers = data.at("samplers");509            if (samplers.is_array()) {510                ctx.params.sampling.samplers = common_sampler_types_from_names(samplers.get<std::vector<std::string>>());511            } else if (samplers.is_string()) {512                ctx.params.sampling.samplers = common_sampler_types_from_chars(samplers.get<std::string>());513            }514        }));515 516    return fields;517}518 519task_params eval_llama_cmpl_schema(520                const llama_vocab * vocab,521                const common_params & params_base,522                const std::vector<llama_logit_bias> & logit_bias_eog,523                const json & data) {524    task_params params;525 526    // Sampling parameter defaults are loaded from the global server context (but individual requests can still override them)527    params.sampling      = params_base.sampling;528    params.speculative   = params_base.speculative;529    params.n_keep        = params_base.n_keep;530    params.n_predict     = params_base.n_predict;531    params.n_cache_reuse = params_base.n_cache_reuse;532    params.cache_prompt  = params_base.cache_prompt;533    params.antiprompt    = params_base.antiprompt;534    params.sse_ping_interval = params_base.sse_ping_interval;535 536    // enabling this will output extra debug information in the HTTP responses from the server537    params.verbose       = params_base.verbosity > 9;538 539    params.chat_parser_params.reasoning_format = params_base.reasoning_format;540 541    // create context and schema542    field_eval_context ctx(params);543    ctx.vocab          = vocab;544    ctx.logit_bias_eog = &logit_bias_eog;545 546    auto schema = make_llama_cmpl_schema(params_base, params);547 548    // eval all fields in the schema549    for (const auto & f : schema) {550        f->eval(ctx, data);551    }552 553    // post-processing554    {555        // if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here556        auto reasoning_format = params.chat_parser_params.reasoning_format;557        params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);558    }559 560    // debugging561    {562        auto budget = params.sampling.reasoning_budget_tokens;563        SRV_DBG("reasoning budget: tokens=%d, generation_prompt='%s', start=%zu toks, end=%zu seqs, forced=%zu toks\n",564                budget, params.sampling.generation_prompt.c_str(),565                params.sampling.reasoning_budget_start.size(),566                params.sampling.reasoning_budget_end.size(),567                params.sampling.reasoning_budget_forced.size());568    }569 570    return params;571}572 573//574// eval() implementations575//576 577static void handle_with_catch(const char * name, std::function<void()> func) {578    try {579        func();580    } catch (const std::exception & e) {581        throw std::invalid_argument(string_format("Field '%s': %s", name, e.what()));582    }583}584 585// treat a null value as absent so clients can send null to request the server default586static bool has_value(const json & data, const char * n) {587    return data.contains(n) && !data.at(n).is_null();588}589 590template <typename T>591void field_num<T>::eval(field_eval_context & ctx, const json & data) {592    for (const auto & n : name) {593        if (has_value(data, n)) {594            handle_with_catch(n, [&]() {595                if (custom_handler) {596                custom_handler(ctx, data);597                } else if (!is_hard_limit) {598                    val = std::max(min, std::min(max, data.at(n).template get<T>()));599                } else {600                    T tmp = data.at(n).template get<T>();601                    if (tmp < min || tmp > max) {602                        throw std::invalid_argument(std::string("Value must be between ") + std::to_string(min) + " <= value <= " + std::to_string(max) + ", but got " + std::to_string(tmp));603                    }604                    val = tmp;605                }606            });607            return;608        }609    }610}611 612void field_str::eval(field_eval_context & ctx, const json & data) {613    GGML_ASSERT(custom_handler);614    for (const auto & n : name) {615        if (has_value(data, n)) {616            handle_with_catch(n, [&]() {617                custom_handler(ctx, data);618            });619            return;620        }621    }622}623 624void field_bool::eval(field_eval_context & ctx, const json & data) {625    for (const auto & n : name) {626        if (has_value(data, n)) {627            handle_with_catch(n, [&]() {628                if (custom_handler) {629                    custom_handler(ctx, data);630                } else {631                    val = data.at(n).get<bool>();632                }633            });634            return;635        }636    }637}638 639void field_json::eval(field_eval_context & ctx, const json & data) {640    GGML_ASSERT(custom_handler);641    for (const auto & n : name) {642        if (has_value(data, n)) {643            handle_with_catch(n, [&]() {644                custom_handler(ctx, data);645            });646            return;647        }648    }649}650 651void field_nested::eval(field_eval_context & ctx, const json & data) {652    for (const auto & n : name) {653        if (data.contains(n) && data.at(n).is_object()) {654            for (auto & f : subfields) {655                f->eval(ctx, data.at(n));656            }657            return;658        }659    }660}661 662} // namespace server_schema663