Felipe97/llama-cpp-compiled

1#include "llama.h"2#include <clocale>3#include <cstdio>4#include <cstring>5#include <iostream>6#include <string>7#include <vector>8 9static void print_usage(int, char ** argv) {10    printf("\nexample usage:\n");11    printf("\n    %s -m model.gguf [-c context_size] [-ngl n_gpu_layers]\n", argv[0]);12    printf("\n");13}14 15int main(int argc, char ** argv) {16    std::setlocale(LC_NUMERIC, "C");17 18    std::string model_path;19    int ngl = 99;20    int n_ctx = 2048;21 22    // parse command line arguments23    for (int i = 1; i < argc; i++) {24        try {25            if (strcmp(argv[i], "-m") == 0) {26                if (i + 1 < argc) {27                    model_path = argv[++i];28                } else {29                    print_usage(argc, argv);30                    return 1;31                }32            } else if (strcmp(argv[i], "-c") == 0) {33                if (i + 1 < argc) {34                    n_ctx = std::stoi(argv[++i]);35                } else {36                    print_usage(argc, argv);37                    return 1;38                }39            } else if (strcmp(argv[i], "-ngl") == 0) {40                if (i + 1 < argc) {41                    ngl = std::stoi(argv[++i]);42                } else {43                    print_usage(argc, argv);44                    return 1;45                }46            } else {47                print_usage(argc, argv);48                return 1;49            }50        } catch (std::exception & e) {51            fprintf(stderr, "error: %s\n", e.what());52            print_usage(argc, argv);53            return 1;54        }55    }56    if (model_path.empty()) {57        print_usage(argc, argv);58        return 1;59    }60 61    // only print errors62    llama_log_set([](enum ggml_log_level level, const char * text, void * /* user_data */) {63        if (level >= GGML_LOG_LEVEL_ERROR) {64            fprintf(stderr, "%s", text);65        }66    }, nullptr);67 68    // load dynamic backends69    ggml_backend_load_all();70 71    // initialize the model72    llama_model_params model_params = llama_model_default_params();73    model_params.n_gpu_layers = ngl;74 75    llama_model * model = llama_model_load_from_file(model_path.c_str(), model_params);76    if (!model) {77        fprintf(stderr , "%s: error: unable to load model\n" , __func__);78        return 1;79    }80 81    const llama_vocab * vocab = llama_model_get_vocab(model);82 83    // initialize the context84    llama_context_params ctx_params = llama_context_default_params();85    ctx_params.n_ctx = n_ctx;86    ctx_params.n_batch = n_ctx;87 88    llama_context * ctx = llama_init_from_model(model, ctx_params);89    if (!ctx) {90        fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__);91        return 1;92    }93 94    // initialize the sampler95    llama_sampler * smpl = llama_sampler_chain_init(llama_sampler_chain_default_params());96    llama_sampler_chain_add(smpl, llama_sampler_init_min_p(0.05f, 1));97    llama_sampler_chain_add(smpl, llama_sampler_init_temp(0.8f));98    llama_sampler_chain_add(smpl, llama_sampler_init_dist(LLAMA_DEFAULT_SEED));99 100    // helper function to evaluate a prompt and generate a response101    auto generate = [&](const std::string & prompt) {102        std::string response;103 104        const bool is_first = llama_memory_seq_pos_max(llama_get_memory(ctx), 0) == -1;105 106        // tokenize the prompt107        const int n_prompt_tokens = -llama_tokenize(vocab, prompt.c_str(), prompt.size(), NULL, 0, is_first, true);108        std::vector<llama_token> prompt_tokens(n_prompt_tokens);109        if (llama_tokenize(vocab, prompt.c_str(), prompt.size(), prompt_tokens.data(), prompt_tokens.size(), is_first, true) < 0) {110            GGML_ABORT("failed to tokenize the prompt\n");111        }112 113        // prepare a batch for the prompt114        llama_batch batch = llama_batch_get_one(prompt_tokens.data(), prompt_tokens.size());115        llama_token new_token_id;116        while (true) {117            // check if we have enough space in the context to evaluate this batch118            int n_ctx = llama_n_ctx(ctx);119            int n_ctx_used = llama_memory_seq_pos_max(llama_get_memory(ctx), 0) + 1;120            if (n_ctx_used + batch.n_tokens > n_ctx) {121                printf("\033[0m\n");122                fprintf(stderr, "context size exceeded\n");123                exit(0);124            }125 126            int ret = llama_decode(ctx, batch);127            if (ret != 0) {128                GGML_ABORT("failed to decode, ret = %d\n", ret);129            }130 131            // sample the next token132            new_token_id = llama_sampler_sample(smpl, ctx, -1);133 134            // is it an end of generation?135            if (llama_vocab_is_eog(vocab, new_token_id)) {136                break;137            }138 139            // convert the token to a string, print it and add it to the response140            char buf[256];141            int n = llama_token_to_piece(vocab, new_token_id, buf, sizeof(buf), 0, true);142            if (n < 0) {143                GGML_ABORT("failed to convert token to piece\n");144            }145            std::string piece(buf, n);146            printf("%s", piece.c_str());147            fflush(stdout);148            response += piece;149 150            // prepare the next batch with the sampled token151            batch = llama_batch_get_one(&new_token_id, 1);152        }153 154        return response;155    };156 157    std::vector<llama_chat_message> messages;158    std::vector<char> formatted(llama_n_ctx(ctx));159    int prev_len = 0;160    while (true) {161        // get user input162        printf("\033[32m> \033[0m");163        std::string user;164        std::getline(std::cin, user);165 166        if (user.empty()) {167            break;168        }169 170        const char * tmpl = llama_model_chat_template(model, /* name */ nullptr);171 172        // add the user input to the message list and format it173        messages.push_back({"user", strdup(user.c_str())});174        int new_len = llama_chat_apply_template(tmpl, messages.data(), messages.size(), true, formatted.data(), formatted.size());175        if (new_len > (int)formatted.size()) {176            formatted.resize(new_len);177            new_len = llama_chat_apply_template(tmpl, messages.data(), messages.size(), true, formatted.data(), formatted.size());178        }179        if (new_len < 0) {180            fprintf(stderr, "failed to apply the chat template\n");181            return 1;182        }183 184        // remove previous messages to obtain the prompt to generate the response185        std::string prompt(formatted.begin() + prev_len, formatted.begin() + new_len);186 187        // generate a response188        printf("\033[33m");189        std::string response = generate(prompt);190        printf("\n\033[0m");191 192        // add the response to the messages193        messages.push_back({"assistant", strdup(response.c_str())});194        prev_len = llama_chat_apply_template(tmpl, messages.data(), messages.size(), false, nullptr, 0);195        if (prev_len < 0) {196            fprintf(stderr, "failed to apply the chat template\n");197            return 1;198        }199    }200 201    // free resources202    for (auto & msg : messages) {203        free(const_cast<char *>(msg.content));204    }205    llama_sampler_free(smpl);206    llama_free(ctx);207    llama_model_free(model);208 209    return 0;210}211