Felipe97/llama-cpp-compiled

1#pragma once2 3#include "llama-memory-hybrid.h"4 5#include <memory>6#include <vector>7 8//9// llama_memory_hybrid_idx10//11 12// llama_memory_hybrid plus a third cache with one indexer key per token, for block-sparse attention (qwen4exp QSA)13// the indexer is a side buffer over the attention cells: same size, padding, streams and slots, so cell j is one token in both14 15class llama_memory_hybrid_idx : public llama_memory_hybrid {16public:17    llama_memory_hybrid_idx(18        const llama_model & model,19                            /* attn */20                ggml_type   type_k,21                ggml_type   type_v,22                     bool   v_trans,23                 uint32_t   kv_size,24                 uint32_t   n_pad,25                 uint32_t   n_swa,26           llama_swa_type   swa_type,27                            /* recurrent */28                ggml_type   type_r,29                ggml_type   type_s,30                 uint32_t   rs_size,31                            /* common */32                 uint32_t   n_seq_max,33                 uint32_t   n_rs_seq,34                     bool   offload,35                     bool   unified,36                            /* layer filters */37    const layer_filter_cb & filter_attn,38    const layer_filter_cb & filter_recr,39                            /* the indexer cache exists only if this is given */40    const layer_filter_cb & filter_idx);41 42    ~llama_memory_hybrid_idx() = default;43 44    //45    // llama_memory_i46    //47 48    llama_memory_context_ptr init_batch(49            llama_batch_allocr & balloc,50            uint32_t n_ubatch,51            bool embd_all) override;52 53    llama_memory_context_ptr init_full() override;54 55    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;56 57    void clear(bool data) override;58 59    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;60    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;61    void seq_keep(llama_seq_id seq_id)                                                          override;62    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;63    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;64 65    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;66 67    // state write/load68 69    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;70    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0)       override;71 72    //73    // llama_memory_hybrid_idx specific API74    //75 76    llama_kv_cache * get_mem_idx() const;   // nullptr when the model carries no indexer77 78    // block-compressed sparse attention (qwen4exp QSA) over the cells of the indexer cache.79    // Blocks cut the position line, not the cell array, so no caller assumes a contiguous layout:80    //   cell_blk  I32 [n_kv, ns]           block each cell belongs to81    //   blk_cells I32 [ratio*n_blocks, ns] cells making up each block82    //   blk_pos   I32 [4*n_blocks*ns]      mrope position rows of each block's first token83    //   bias      F32 [n_kv, n_tokens/ns, ns] -inf where invisible, large where always visible84    // blk_bias asks for the bias per block instead: [n_blocks, n_tokens/ns, ns]85    // the caller then adds the attention mask, the only part of the bias that varies within a block86    void set_input_qsa(ggml_tensor * cell_blk, ggml_tensor * blk_cells, ggml_tensor * blk_pos,87                       ggml_tensor * bias, const llama_ubatch * ubatch, uint32_t ratio,88                       bool blk_bias) const;89 90private:91    // forget seq_id (all of it if seq_id < 0) in every cache at once, so a failed restore cannot leave the caches out of step92    // seq_id < 0 drops the whole context, as the caches themselves do on a failed restore93    void state_drop(llama_seq_id seq_id);94 95    // the indexer cache holds one key head per layer, so it needs its own hparams:96    // llama_kv_cache keeps a reference to what it is given97    llama_hparams hparams_idx;98 99    const std::unique_ptr<llama_kv_cache> mem_idx;100};101 102class llama_memory_hybrid_idx_context : public llama_memory_hybrid_context {103public:104    using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;105 106    // used for errors107    explicit llama_memory_hybrid_idx_context(llama_memory_status status);108 109    // used to create a full-cache context110    explicit llama_memory_hybrid_idx_context(llama_memory_hybrid_idx * mem);111 112    // used to create an update context113    llama_memory_hybrid_idx_context(114            llama_memory_hybrid_idx * mem,115                      llama_context * lctx,116                               bool   optimize);117 118    // used to create a batch processing context from a batch119    llama_memory_hybrid_idx_context(120            llama_memory_hybrid_idx * mem,121                    slot_info_vec_t   sinfos_attn,122                    slot_info_vec_t   sinfos_idx,123          std::vector<llama_ubatch>   ubatches);124 125    ~llama_memory_hybrid_idx_context() = default;126 127    //128    // llama_memory_context_i129    //130 131    bool next()  override;132    bool apply() override;133 134    //135    // llama_memory_hybrid_idx_context specific API136    //137 138    // nullptr with no indexer139    const llama_kv_cache_context * get_idx() const;140 141    // streams in the current slot info, the `ns` of get_k/get_v; 1 if unified142    uint32_t get_n_stream() const;143 144    void set_input_qsa(ggml_tensor * cell_blk, ggml_tensor * blk_cells, ggml_tensor * blk_pos,145                       ggml_tensor * bias, const llama_ubatch * ubatch, uint32_t ratio,146                       bool blk_bias) const;147 148private:149    const llama_memory_hybrid_idx * mem = nullptr;150 151    // streams per ubatch, read from the slot infos before ctx_idx takes them152    // declared first, so it is initialised while sinfos_idx is still intact153    const std::vector<uint32_t> ns_ubatch;154 155    // null unless the model has an indexer156    const llama_memory_context_ptr ctx_idx;157 158    // mirrors the base class's ubatch cursor, which is private there159    size_t i_cur = 0;160};161