Felipe97/llama-cpp-compiled

1#pragma once2 3#include "llama.h"4#include "common.h"5 6struct common_speculative;7 8// comma separated list the provided types9std::string common_speculative_type_name_str(const std::vector<enum common_speculative_type> & types);10 11// comma separated list of all types12const char * common_speculative_all_types_str();13 14// parse user provided types15std::vector<enum common_speculative_type> common_speculative_types_from_names(const std::vector<std::string> & names);16 17// infer the spec types from the GGUF metadata of a draft model; empty if unknown18std::vector<enum common_speculative_type> common_speculative_types_from_gguf(const std::string & path);19 20// convert string to type21enum common_speculative_type common_speculative_type_from_name(const std::string & name);22 23// convert type to string24std::string common_speculative_type_to_str(enum common_speculative_type type);25 26// return the max number of draft tokens based on the speculative parameters27int32_t common_speculative_n_max(const common_params_speculative * spec);28 29// return the max number of draft tokens from the initialized implementations30int32_t common_speculative_n_max(const common_speculative * spec);31 32// validate and resolve the unconditional synthetic acceptance rates33std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max);34 35// return the conditional synthetic acceptance probabilities36const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec);37 38common_params common_base_params_to_speculative(const common_params & params);39 40struct common_speculative_output_limits {41    int32_t total;42    int32_t per_seq;43};44 45// return the output limits needed for speculative decoding46common_speculative_output_limits common_speculative_get_output_limits(47        int32_t n_batch, int32_t n_parallel, int32_t n_draft);48 49common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq);50 51void common_speculative_free(common_speculative * spec);52 53struct common_speculative_draft_params {54    // this flag is used to chain the drafts through all the available implementations55    // after the first successful draft from an implementation, we set it56    //   to false to prevent further drafts for that sequence57    // at the end of the draft() call, all drafting flags will be reset to false58    bool drafting = false;59 60    // overrides individual configurations (-1 disabled)61    // can be used to constraint the max draft based on the remaining context size62    int32_t n_max = -1;63 64    llama_pos   pos0;65    llama_token id_last;66 67    // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls68    const llama_tokens * prompt;69 70    // the generated draft from the last _draft() call71    llama_tokens * result;72};73 74common_speculative_draft_params & common_speculative_get_draft_params(common_speculative * spec, llama_seq_id seq_id);75 76// optionally call once at the beginning of a new generation77void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, const llama_tokens & prompt);78 79// process the batch and update the internal state of the speculative context80bool common_speculative_process(common_speculative * spec, const llama_batch & batch);81 82// generate drafts for the sequences specified with `common_speculative_get_draft_params`83void common_speculative_draft(common_speculative * spec);84 85// informs the speculative context that n_accepted tokens were accepted by the target model86void common_speculative_accept(common_speculative * spec, llama_seq_id, uint16_t n_accepted);87 88// (optional) get/set internal state89bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector<uint8_t> & data);90void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector<uint8_t> & data);91 92// print statistics about the speculative decoding93void common_speculative_print_stats(const common_speculative * spec);94 95struct common_speculative_deleter {96    void operator()(common_speculative * s) { common_speculative_free(s); }97};98 99typedef std::unique_ptr<common_speculative, common_speculative_deleter> common_speculative_ptr;100 101struct common_speculative_init_result {102    common_speculative_init_result(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt);103    ~common_speculative_init_result();104 105    llama_model   * model();106    llama_context * context();107 108private:109    struct impl;110    std::unique_ptr<impl> pimpl;111};112 113using common_speculative_init_result_ptr = std::unique_ptr<common_speculative_init_result>;114 115common_speculative_init_result_ptr common_speculative_init_from_params(common_params & params, llama_model * model_tgt, llama_context * ctx_tgt);116