Felipe97/llama-cpp-compiled

1#include "arg.h"2#include "common.h"3#include "log.h"4#include "llama.h"5 6#include <algorithm>7#include <clocale>8#include <cstdio>9#include <string>10#include <vector>11 12static void print_usage(int, char ** argv) {13    LOG("\nexample usage:\n");14    LOG("\n    %s -m model.gguf -c 2048 -b 2048 -ub 512 -npp 128,256,512 -ntg 128,256 -npl 1,2,4,8,16,32 [-pps]\n", argv[0]);15    LOG("\n");16}17 18// satisfies -Wmissing-declarations19int llama_batched_bench(int argc, char ** argv);20 21int llama_batched_bench(int argc, char ** argv) {22    std::setlocale(LC_NUMERIC, "C");23 24    common_params params;25 26    common_init();27 28    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_BENCH, print_usage)) {29        return 1;30    }31 32    int is_pp_shared   = params.is_pp_shared;33    int is_tg_separate = params.is_tg_separate;34 35    std::vector<int> n_pp = params.n_pp;36    std::vector<int> n_tg = params.n_tg;37    std::vector<int> n_pl = params.n_pl;38 39    // init LLM40 41    llama_backend_init();42    llama_numa_init(params.numa);43 44    // initialize the model45 46    llama_model_params model_params = common_model_params_to_llama(params);47 48    llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);49 50    if (model == NULL) {51        fprintf(stderr , "%s: error: unable to load model\n" , __func__);52        return 1;53    }54 55    llama_context_params ctx_params = common_context_params_to_llama(params);56 57    // ensure enough sequences are available58    ctx_params.n_seq_max = n_pl.empty() ? 1 : *std::max_element(n_pl.begin(), n_pl.end());59 60    llama_context * ctx = llama_init_from_model(model, ctx_params);61 62    if (ctx == NULL) {63        fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__);64        llama_model_free(model);65        return 1;66    }67 68    const llama_vocab * vocab   = llama_model_get_vocab(model);69    const int32_t       n_vocab = llama_vocab_n_tokens(vocab);70 71    const auto get_token_rand = [n_vocab]() -> llama_token {72        return std::rand() % n_vocab;73    };74 75    auto * mem = llama_get_memory(ctx);76 77    const int32_t n_kv_max = llama_n_ctx(ctx);78 79    llama_batch batch = llama_batch_init(n_kv_max, 0, 1);80 81    // decode in batches of ctx_params.n_batch tokens82    auto decode_helper = [](llama_context * ctx, llama_batch & batch, int32_t n_batch, bool synchronize) {83        for (int32_t i = 0; i < batch.n_tokens; i += n_batch) {84            const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);85 86            llama_batch batch_view = {87                n_tokens,88                batch.token    + i,89                nullptr,90                batch.pos      + i,91                batch.n_seq_id + i,92                batch.seq_id   + i,93                batch.logits   + i,94            };95 96            const int ret = llama_decode(ctx, batch_view);97            if (ret != 0) {98                LOG_ERR("failed to decode the batch, n_batch = %d, ret = %d\n", n_batch, ret);99                return false;100            }101 102            if (synchronize) {103                llama_synchronize(ctx);104            }105        }106 107        return true;108    };109 110    // warm up111    {112        for (int i = 0; i < 16; ++i) {113            common_batch_add(batch, get_token_rand(), i, { 0 }, false);114        }115 116        if (!decode_helper(ctx, batch, ctx_params.n_batch, true)) {117            LOG_ERR("%s: llama_decode() failed\n", __func__);118            llama_free(ctx);119            llama_model_free(model);120            return 1;121        }122    }123 124    if (!params.batched_bench_output_jsonl) {125        LOG("\n");126        LOG("%s: n_kv_max = %d, n_batch = %d, n_ubatch = %d, flash_attn = %d, is_pp_shared = %d, is_tg_separate = %d, n_gpu_layers = %d, n_threads = %u, n_threads_batch = %u\n", __func__, n_kv_max, params.n_batch, params.n_ubatch, int(params.flash_attn_type), is_pp_shared, is_tg_separate, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch);127        LOG("\n");128        LOG("|%6s | %6s | %4s | %6s | %8s | %8s | %8s | %8s | %8s | %8s |\n", "PP", "TG", "B", "N_KV", "T_PP s", "S_PP t/s", "T_TG s", "S_TG t/s", "T s", "S t/s");129        LOG("|%6s-|-%6s-|-%4s-|-%6s-|-%8s-|-%8s-|-%8s-|-%8s-|-%8s-|-%8s-|\n", "------", "------", "----", "------", "--------", "--------", "--------", "--------", "--------", "--------");130    }131 132    for (        int i_pp = 0; i_pp < (int) n_pp.size(); ++i_pp) {133        for (    int i_tg = 0; i_tg < (int) n_tg.size(); ++i_tg) {134            for (int i_pl = 0; i_pl < (int) n_pl.size(); ++i_pl) {135                const int pp = n_pp[i_pp];136                const int tg = n_tg[i_tg];137                const int pl = n_pl[i_pl];138 139                const int n_ctx_req = is_pp_shared ? (params.kv_unified ? pp : pl*pp) + pl*tg : pl*(pp + tg);140 141                if (n_ctx_req > n_kv_max) {142                    continue;143                }144 145                common_batch_clear(batch);146 147                for (int j = 0; j < (is_pp_shared ? 1 : pl); ++j) {148                    for (int i = 0; i < pp; ++i) {149                        common_batch_add(batch, get_token_rand(), i, { j }, i == pp - 1);150                    }151                }152 153                llama_memory_clear(mem, false);154 155                const auto t_pp_start = ggml_time_us();156 157                if (!decode_helper(ctx, batch, ctx_params.n_batch, false)) {158                    LOG_ERR("%s: llama_decode() failed\n", __func__);159                    llama_free(ctx);160                    llama_model_free(model);161                    return 1;162                }163 164                llama_synchronize(ctx);165 166                const auto t_pp_end = ggml_time_us();167 168                if (is_pp_shared) {169                    for (int32_t i = 1; i < pl; ++i) {170                        llama_memory_seq_cp(mem, 0, i, -1, -1);171                    }172 173                    if (!params.kv_unified) {174                        // run one dummy token to apply the memory copy175                        common_batch_clear(batch);176                        common_batch_add(batch, get_token_rand(), pp + 0, { 0 }, true);177                        if (!decode_helper(ctx, batch, ctx_params.n_batch, true)) {178                            LOG_ERR("%s: llama_decode() failed\n", __func__);179                            llama_free(ctx);180                            llama_model_free(model);181                            return 1;182                        }183                        llama_memory_seq_rm(mem, 0, pp, -1);184                    }185                }186 187                const auto t_tg_start = ggml_time_us();188 189                if (is_tg_separate) {190                    // decode pattern:191                    // 0 0 0 ... 1 1 1 ... 2 2 2 ... 3 3 3 ...192                    for (int j = 0; j < pl; ++j) {193                        for (int i = 0; i < tg; ++i) {194                            common_batch_clear(batch);195 196                            common_batch_add(batch, get_token_rand(), pp + i, { j }, true);197 198                            if (!decode_helper(ctx, batch, ctx_params.n_batch, true)) {199                                LOG_ERR("%s: llama_decode() failed\n", __func__);200                                llama_free(ctx);201                                llama_model_free(model);202                                return 1;203                            }204                        }205                    }206                } else {207                    // decode pattern:208                    // 0123 0123 0123 ...209                    for (int i = 0; i < tg; ++i) {210                        common_batch_clear(batch);211 212                        for (int j = 0; j < pl; ++j) {213                            common_batch_add(batch, get_token_rand(), pp + i, { j }, true);214                        }215 216                        if (!decode_helper(ctx, batch, ctx_params.n_batch, true)) {217                            LOG_ERR("%s: llama_decode() failed\n", __func__);218                            llama_free(ctx);219                            llama_model_free(model);220                            return 1;221                        }222                    }223                }224 225                const auto t_tg_end = ggml_time_us();226 227                const int32_t n_kv = n_ctx_req;228 229                const float t_pp = (t_pp_end - t_pp_start) / 1000000.0f;230                const float t_tg = (t_tg_end - t_tg_start) / 1000000.0f;231                const float t    = t_pp + t_tg;232 233                const float speed_pp = is_pp_shared ? pp / t_pp : pl*pp / t_pp;234                const float speed_tg = pl*tg / t_tg;235                const float speed    = ((is_pp_shared ? pp : pl*pp) + pl*tg) / t;236 237                if(params.batched_bench_output_jsonl) {238                    LOG(239                        "{\"n_kv_max\": %d, \"n_batch\": %d, \"n_ubatch\": %d, \"flash_attn\": %d, \"is_pp_shared\": %d, \"n_gpu_layers\": %d, \"n_threads\": %u, \"n_threads_batch\": %u, "240                        "\"pp\": %d, \"tg\": %d, \"pl\": %d, \"n_kv\": %d, \"t_pp\": %f, \"speed_pp\": %f, \"t_tg\": %f, \"speed_tg\": %f, \"t\": %f, \"speed\": %f}\n",241                        n_kv_max, params.n_batch, params.n_ubatch, int(params.flash_attn_type), params.is_pp_shared, params.n_gpu_layers, ctx_params.n_threads, ctx_params.n_threads_batch,242                        pp, tg, pl, n_kv, t_pp, speed_pp, t_tg, speed_tg, t, speed243                    );244                } else {245                    LOG("|%6d | %6d | %4d | %6d | %8.3f | %8.2f | %8.3f | %8.2f | %8.3f | %8.2f |\n", pp, tg, pl, n_kv, t_pp, speed_pp, t_tg, speed_tg, t, speed);246                }247            }248        }249    }250 251    LOG("\n");252    llama_perf_context_print(ctx);253 254    llama_batch_free(batch);255 256    llama_free(ctx);257    llama_model_free(model);258 259    llama_backend_free();260 261    return 0;262}263