Felipe97/llama-cpp-compiled

1#include "llama-adapter.h"2 3#include "llama-impl.h"4#include "llama-mmap.h"5#include "llama-model.h"6 7#include <map>8#include <cassert>9#include <cerrno>10#include <cstring>11#include <sstream>12#include <stdexcept>13 14// vec15 16ggml_tensor * llama_adapter_cvec::tensor_for(int il) const {17    if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) {18        return nullptr;19    }20 21    return tensors[il];22}23 24ggml_tensor * llama_adapter_cvec::apply_to(ggml_context * ctx, ggml_tensor * cur, int  il) const {25    ggml_tensor * layer_dir = tensor_for(il);26    if (layer_dir != nullptr) {27        cur = ggml_add(ctx, cur, layer_dir);28    }29 30    return cur;31}32 33bool llama_adapter_cvec::init(const llama_model & model) {34    const auto & hparams = model.hparams;35 36    GGML_ASSERT(tensors.empty());37    GGML_ASSERT(ctxs.empty());38    GGML_ASSERT(bufs.empty());39 40    // create a context for each buffer type41    std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;42    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {43        auto it = ctx_map.find(buft);44        if (it == ctx_map.end()) {45            ggml_init_params params = {46                /*.mem_size   =*/ hparams.n_layer()*ggml_tensor_overhead(),47                /*.mem_buffer =*/ NULL,48                /*.no_alloc   =*/ true,49            };50 51            ggml_context * ctx = ggml_init(params);52            if (!ctx) {53                return nullptr;54            }55 56            ctx_map[buft] = ctx;57            ctxs.emplace_back(ctx);58 59            return ctx;60        }61 62        return it->second;63    };64 65    // make tensors66    tensors.reserve(hparams.n_layer());67    tensors.push_back(nullptr); // there's never a tensor for layer 068    for (size_t il = 1; il < hparams.n_layer(); il++) {69        ggml_backend_buffer_type_t buft = model.select_buft(il);70        ggml_context * ctx = ctx_for_buft(buft);71        if (!ctx) {72            LLAMA_LOG_ERROR("%s: failed to allocate context for control vector\n", __func__);73            return false;74        }75        ggml_tensor * tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hparams.n_embd);76        tensors.push_back(tensor);77    }78 79    // allocate tensors / buffers and zero80    bufs.reserve(ctx_map.size());81    for (auto it : ctx_map) {82        ggml_backend_buffer_type_t buft = it.first;83        ggml_context * ctx = it.second;84        ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);85        if (!buf) {86            LLAMA_LOG_ERROR("%s: failed to allocate buffer for control vector\n", __func__);87            return false;88        }89        ggml_backend_buffer_clear(buf, 0);90        bufs.emplace_back(buf);91    }92 93    return true;94}95 96bool llama_adapter_cvec::apply(97        const llama_model & model,98        const float * data,99        size_t len,100        int32_t n_embd,101        int32_t il_start,102        int32_t il_end) {103    const auto & hparams = model.hparams;104 105    if (data == nullptr) {106        // disable the current control vector (but leave allocated for later)107        layer_start = -1;108        layer_end   = -1;109        return true;110    }111 112    if (n_embd != (int) hparams.n_embd) {113        LLAMA_LOG_ERROR("%s: control vector n_embd does not match model\n", __func__);114        return false;115    }116 117    if (tensors.empty()) {118        if (!init(model)) {119            return false;120        }121    }122 123    layer_start = il_start;124    layer_end   = il_end;125 126    for (size_t il = 1; il < hparams.n_layer(); il++) {127        assert(tensors[il] != nullptr);128 129        const size_t off = n_embd * (il - 1); // buffer doesn't have data for layer 0, since it's never present130        if (off + n_embd <= len) {131            ggml_backend_tensor_set(tensors[il], data + off, 0, n_embd * ggml_element_size(tensors[il]));132        }133    }134 135    return true;136}137 138// lora139 140llama_adapter_lora_weight * llama_adapter_lora::get_weight(ggml_tensor * w) {141    const std::string name(w->name);142 143    const auto pos = ab_map.find(name);144    if (pos != ab_map.end()) {145        return &pos->second;146    }147 148    return nullptr;149}150 151static void llama_adapter_lora_init_impl(llama_model & model, FILE * file, llama_adapter_lora & adapter) {152    ggml_context * ctx_init;153    gguf_init_params meta_gguf_params = {154        /* .no_alloc = */ true,155        /* .ctx      = */ &ctx_init,156    };157 158    gguf_context_ptr ctx_gguf { gguf_init_from_file_ptr(file, meta_gguf_params) };159    if (!ctx_gguf) {160        throw std::runtime_error("failed to load lora adapter from file");161    }162 163    ggml_context_ptr ctx { ctx_init };164 165    // must come after gguf_init_from_file_ptr, the llama_file constructor moves the file position166    llama_file gguf_file(file);167 168    // check metadata169    {170        const gguf_context * gguf_ctx = ctx_gguf.get();171 172        LLAMA_LOG_INFO("%s: Dumping metadata keys/values.\n", __func__);173 174        // get metadata as string175        for (int i = 0; i < gguf_get_n_kv(gguf_ctx); i++) {176            gguf_type type = gguf_get_kv_type(gguf_ctx, i);177            const std::string type_name =178                type == GGUF_TYPE_ARRAY179                ? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(gguf_ctx, i)), gguf_get_arr_n(gguf_ctx, i))180                : gguf_type_name(type);181            const char * name = gguf_get_key(gguf_ctx, i);182            const std::string value = gguf_kv_to_str(gguf_ctx, i);183 184            if (type != GGUF_TYPE_ARRAY) {185                adapter.gguf_kv.emplace(name, value);186            }187 188            const size_t MAX_VALUE_LEN = 40;189            std::string print_value = value.size() > MAX_VALUE_LEN ? format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str()) : value;190            replace_all(print_value, "\n", "\\n");191 192            LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), print_value.c_str());193        }194 195        auto get_kv_str = [&](const std::string & key) -> std::string {196            int id = gguf_find_key(gguf_ctx, key.c_str());197            return id < 0 ? "" : std::string(gguf_get_val_str(gguf_ctx, id));198        };199        auto get_kv_f32 = [&](const std::string & key) -> float {200            int id = gguf_find_key(gguf_ctx, key.c_str());201            return id < 0 ? 0.0f : gguf_get_val_f32(gguf_ctx, id);202        };203        LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);204 205        auto general_type = get_kv_str(llm_kv(LLM_KV_GENERAL_TYPE));206        if (general_type != "adapter") {207            throw std::runtime_error("expect general.type to be 'adapter', but got: " + general_type);208        }209 210        auto general_arch_str = get_kv_str(llm_kv(LLM_KV_GENERAL_ARCHITECTURE));211        auto general_arch = llm_arch_from_string(general_arch_str);212        if (general_arch != model.arch) {213            throw std::runtime_error("model arch and LoRA arch mismatch");214        }215 216        auto adapter_type = get_kv_str(llm_kv(LLM_KV_ADAPTER_TYPE));217        if (adapter_type != "lora") {218            throw std::runtime_error("expect adapter.type to be 'lora', but got: " + adapter_type);219        }220 221        adapter.alpha = get_kv_f32(llm_kv(LLM_KV_ADAPTER_LORA_ALPHA));222 223        // parse alora invocation sequence vector224        const auto & key = llm_kv(LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS);225        const int kid = gguf_find_key(ctx_gguf.get(), key.c_str());226        if (kid >= 0) {227            if (gguf_get_kv_type(ctx_gguf.get(), kid) != GGUF_TYPE_ARRAY) {228                throw std::runtime_error("invalid gguf type for " + key);229            }230            const auto arr_type = gguf_get_arr_type(ctx_gguf.get(), kid);231            if (arr_type != GGUF_TYPE_UINT32) {232                throw std::runtime_error("invalid gguf element type for " + key);233            }234            const size_t seq_len = gguf_get_arr_n(ctx_gguf.get(), kid);235            const void * data = gguf_get_arr_data(ctx_gguf.get(), kid);236            adapter.alora_invocation_tokens.resize(seq_len);237            std::copy(238                (const llama_token *)data,239                (const llama_token *)data + seq_len,240                adapter.alora_invocation_tokens.begin());241        }242    }243 244    int n_tensors = gguf_get_n_tensors(ctx_gguf.get());245 246    // contexts for each buffer type247    std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;248    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {249        auto it = ctx_map.find(buft);250        if (it == ctx_map.end()) {251            // add a new context252            ggml_init_params params = {253                /*.mem_size   =*/ n_tensors*ggml_tensor_overhead(),254                /*.mem_buffer =*/ NULL,255                /*.no_alloc   =*/ true,256            };257            ggml_context * buft_ctx = ggml_init(params);258            if (!buft_ctx) {259                return nullptr;260            }261            ctx_map[buft] = buft_ctx;262            adapter.ctxs.emplace_back(buft_ctx);263            return buft_ctx;264        };265        return it->second;266    };267 268    // bundle lora_a and lora_b into pairs269    std::map<std::string, llama_adapter_lora_weight> ab_map;270    auto str_endswith = [](const std::string & str, const std::string & suffix) {271        return str.size() >= suffix.size() && str.compare(str.size()-suffix.size(), suffix.size(), suffix) == 0;272    };273 274    for (ggml_tensor * cur = ggml_get_first_tensor(ctx.get()); cur; cur = ggml_get_next_tensor(ctx.get(), cur)) {275        std::string name(cur->name);276        if (str_endswith(name, ".lora_a")) {277            replace_all(name, ".lora_a", "");278            if (ab_map.find(name) == ab_map.end()) {279                ab_map[name] = llama_adapter_lora_weight(cur, nullptr);280            } else {281                ab_map[name].a = cur;282            }283        } else if (str_endswith(name, ".lora_b")) {284            replace_all(name, ".lora_b", "");285            if (ab_map.find(name) == ab_map.end()) {286                ab_map[name] = llama_adapter_lora_weight(nullptr, cur);287            } else {288                ab_map[name].b = cur;289            }290        } else if (str_endswith(name, "_norm.weight")) {291            // TODO: add support for norm vector292            // for now, we don't really care because most adapters still work fine without it293            continue;294        } else {295            throw std::runtime_error("LoRA tensor '" + name + "' has unexpected suffix");296        }297    }298 299    // get extra buffer types of the CPU300    // TODO: a more general solution for non-CPU extra buft should be implemented in the future301    //       ref: https://github.com/ggml-org/llama.cpp/pull/12593#pullrequestreview-2718659948302    std::vector<ggml_backend_buffer_type_t> buft_extra;303    {304        auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);305        if (!cpu_dev) {306            throw std::runtime_error(format("%s: no CPU backend found", __func__));307        }308        auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);309 310        auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)311            ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_dev_get_extra_bufts");312 313        if (ggml_backend_dev_get_extra_bufts_fn) {314            ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);315            while (extra_bufts && *extra_bufts) {316                buft_extra.emplace_back(*extra_bufts);317                ++extra_bufts;318            }319        }320    }321 322    // add tensors323    for (auto & it : ab_map) {324        const std::string & name = it.first;325        llama_adapter_lora_weight & w = it.second;326        bool is_token_embd = str_endswith(name, "token_embd.weight");327 328        if (!w.a || !w.b) {329            throw std::runtime_error("LoRA tensor pair for '" + name + "' is missing one component");330        }331 332        // device buft and device ctx333        const auto * model_tensor = model.get_tensor(name.c_str());334        if (!model_tensor) {335            throw std::runtime_error("LoRA tensor '" + name + "' does not exist in base model (hint: maybe wrong base model?)");336        }337 338        auto * buft = ggml_backend_buffer_get_type(model_tensor->buffer);339 340        // do not load loras to extra buffer types (i.e. bufts for repacking) -> use the CPU in that case341        for (auto & ex : buft_extra) {342            if (ex == buft) {343                LLAMA_LOG_WARN("%s: lora for '%s' cannot use buft '%s', fallback to CPU\n", __func__, model_tensor->name, ggml_backend_buft_name(buft));344 345                auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);346                if (!cpu_dev) {347                    throw std::runtime_error(format("%s: no CPU backend found", __func__));348                }349                buft = ggml_backend_dev_buffer_type(cpu_dev);350 351                break;352            }353        }354 355        LLAMA_LOG_DEBUG("%s: lora for '%s' -> '%s'\n", __func__, model_tensor->name, ggml_backend_buft_name(buft));356 357        ggml_context * dev_ctx = ctx_for_buft(buft);358        // validate tensor shape359        if (is_token_embd) {360            // expect B to be non-transposed, A and B are flipped; see llm_build_inp_embd()361            if (model_tensor->ne[0] != w.b->ne[1] || model_tensor->ne[1] != w.a->ne[1]) {362                throw std::runtime_error("tensor '" + name + "' has incorrect shape (hint: maybe wrong base model?)");363            }364        } else {365            if (model_tensor->ne[0] != w.a->ne[0] || model_tensor->ne[1] != w.b->ne[1]) {366                throw std::runtime_error("tensor '" + name + "' has incorrect shape (hint: maybe wrong base model?)");367            }368            if (w.a->ne[1] != w.b->ne[0]) {369                throw std::runtime_error("lora_a tensor is not transposed (hint: adapter from \"finetune\" example is no longer supported)");370            }371        }372 373        // save tensor to adapter374        ggml_tensor * tensor_a = ggml_dup_tensor(dev_ctx, w.a);375        ggml_tensor * tensor_b = ggml_dup_tensor(dev_ctx, w.b);376        ggml_set_name(tensor_a, w.a->name);377        ggml_set_name(tensor_b, w.b->name);378        adapter.ab_map[name] = llama_adapter_lora_weight(tensor_a, tensor_b);379    }380 381    // allocate tensors / buffers and zero382    {383        adapter.ctxs.reserve(ctx_map.size());384        adapter.bufs.reserve(ctx_map.size());385        for (auto & it : ctx_map) {386            ggml_backend_buffer_type_t buft = it.first;387            ggml_context * ctx_dev = it.second;388            ggml_backend_buffer_ptr buf { ggml_backend_alloc_ctx_tensors_from_buft(ctx_dev, buft) };389            if (!buf) {390                throw std::runtime_error("failed to allocate buffer for lora adapter\n");391            }392            LLAMA_LOG_INFO("%s: %10s LoRA buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf.get()), ggml_backend_buffer_get_size(buf.get())/1024.0/1024.0);393            adapter.bufs.emplace_back(std::move(buf));394        }395    }396 397    // set tensor data398    {399        std::vector<uint8_t> read_buf;400        auto set_tensor = [&](ggml_tensor * orig, ggml_tensor * dev) {401            const size_t offs = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), gguf_find_tensor(ctx_gguf.get(), orig->name));402            const size_t size = ggml_nbytes(orig);403            if (offs + size < offs || offs + size > gguf_file.size()) {404                throw std::runtime_error(format("LoRA tensor '%s' data is not within the file bounds, file is corrupted or incomplete", orig->name));405            }406            read_buf.resize(size);407            gguf_file.seek(offs, SEEK_SET);408            gguf_file.read_raw(read_buf.data(), size);409            ggml_backend_tensor_set(dev, read_buf.data(), 0, size);410        };411        for (auto & it : adapter.ab_map) {412            auto orig = ab_map[it.first];413            auto dev  = it.second;414            set_tensor(orig.a, dev.a);415            set_tensor(orig.b, dev.b);416        }417    }418 419    // register adapter with model420    model.loras.insert(&adapter);421 422    LLAMA_LOG_INFO("%s: loaded %zu tensors from lora file\n", __func__, adapter.ab_map.size()*2);423}424 425llama_adapter_lora * llama_adapter_lora_init(llama_model * model, const char * path_lora) {426    LLAMA_LOG_INFO("%s: loading lora adapter from '%s' ...\n", __func__, path_lora);427 428    FILE * file = ggml_fopen(path_lora, "rb");429    if (!file) {430        LLAMA_LOG_ERROR("%s: failed to open '%s': %s\n", __func__, path_lora, strerror(errno));431        return nullptr;432    }433 434    llama_adapter_lora * adapter = llama_adapter_lora_init_from_file_ptr(model, file);435    fclose(file);436 437    return adapter;438}439 440llama_adapter_lora * llama_adapter_lora_init_from_file_ptr(llama_model * model, FILE * file) {441    if (!file) {442        LLAMA_LOG_ERROR("%s: file is NULL\n", __func__);443        return nullptr;444    }445 446    llama_adapter_lora * adapter = new llama_adapter_lora(model);447 448    try {449        llama_adapter_lora_init_impl(*model, file, *adapter);450        return adapter;451    } catch (const std::exception & err) {452        LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what());453 454        delete adapter;455    }456 457    return nullptr;458}459 460int32_t llama_adapter_meta_val_str(const llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size) {461    const auto & it = adapter->gguf_kv.find(key);462    if (it == adapter->gguf_kv.end()) {463        if (buf_size > 0) {464            buf[0] = '\0';465        }466        return -1;467    }468    return snprintf(buf, buf_size, "%s", it->second.c_str());469}470 471int32_t llama_adapter_meta_count(const llama_adapter_lora * adapter) {472    return (int)adapter->gguf_kv.size();473}474 475int32_t llama_adapter_meta_key_by_index(const llama_adapter_lora * adapter, int i, char * buf, size_t buf_size) {476    if (i < 0 || i >= (int)adapter->gguf_kv.size()) {477        if (buf_size > 0) {478            buf[0] = '\0';479        }480        return -1;481    }482    auto it = adapter->gguf_kv.begin();483    std::advance(it, i);484    return snprintf(buf, buf_size, "%s", it->first.c_str());485}486 487int32_t llama_adapter_meta_val_str_by_index(const llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size) {488    if (i < 0 || i >= (int)adapter->gguf_kv.size()) {489        if (buf_size > 0) {490            buf[0] = '\0';491        }492        return -1;493    }494    auto it = adapter->gguf_kv.begin();495    std::advance(it, i);496    return snprintf(buf, buf_size, "%s", it->second.c_str());497}498 499void llama_adapter_lora_free(llama_adapter_lora * adapter) {500    if (adapter == nullptr) {501        return;502    }503 504    if (adapter->model != nullptr) {505        adapter->model->loras.erase(adapter);506        adapter->model = nullptr;507    }508 509    delete adapter;510}511 512uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter) {513    if (!adapter) {514        return 0;515    }516    return adapter->alora_invocation_tokens.size();517}518 519const llama_token * llama_adapter_get_alora_invocation_tokens(const llama_adapter_lora * adapter) {520    GGML_ASSERT(adapter);521    return adapter->alora_invocation_tokens.data();522}523