Felipe97/llama-cpp-compiled
01.1k
1// A basic application simulating a server with multiple clients.2// The clients submit requests to the server and they are processed in parallel.3 4#include "arg.h"5#include "common.h"6#include "sampling.h"7#include "log.h"8#include "llama.h"9 10#include <algorithm>11#include <clocale>12#include <cmath>13#include <cstdio>14#include <random>15#include <string>16#include <vector>17#include <ctime>18 19// trim whitespace from the beginning and end of a string20static std::string trim(const std::string & str) {21 size_t start = 0;22 size_t end = str.size();23 24 while (start < end && isspace(str[start])) {25 start += 1;26 }27 28 while (end > start && isspace(str[end - 1])) {29 end -= 1;30 }31 32 return str.substr(start, end - start);33}34 35static std::string k_system =36R"(Transcript of a never ending dialog, where the User interacts with an Assistant.37The Assistant is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.38 39User:40Recommend a nice restaurant in the area.41Assistant:42I recommend the restaurant "The Golden Duck". It is a 5 star restaurant with a great view of the city. The food is delicious and the service is excellent. The prices are reasonable and the portions are generous. The restaurant is located at 123 Main Street, New York, NY 10001. The phone number is (212) 555-1234. The hours are Monday through Friday from 11:00 am to 10:00 pm. The restaurant is closed on Saturdays and Sundays.43User:44Who is Richard Feynman?45Assistant:46Richard Feynman was an American physicist who is best known for his work in quantum mechanics and particle physics. He was awarded the Nobel Prize in Physics in 1965 for his contributions to the development of quantum electrodynamics. He was a popular lecturer and author, and he wrote several books, including "Surely You're Joking, Mr. Feynman!" and "What Do You Care What Other People Think?".47)";48 49static std::vector<std::string> k_questions = {50 "What is the tallest mountain in the world?",51 "Who was the first person to win two Nobel Prizes?",52 "Which country invented paper?",53 "What organ is primarily responsible for pumping blood throughout the body?",54 "Which planet is known for its prominent ring system?",55 "Who directed the movie 'Inception'?",56 "What is the freezing point of water in Fahrenheit?",57 "Which animal is known to have the longest lifespan?",58 "What language has the most native speakers worldwide?",59 "What is the capital city of Canada?",60 "Who is credited with inventing the World Wide Web?",61 "Which metal is liquid at room temperature?",62 "What is the term for an animal that eats both plants and meat?",63 "Who painted 'The Starry Night'?",64 "What gas do humans exhale that plants use for photosynthesis?",65 "What year did World War II end?",66 "Which continent has the most countries?",67 "Who wrote the novel 'Frankenstein'?",68 "What does DNA stand for?",69 "What is the main ingredient in traditional Japanese miso soup?"70};71 72static std::vector<std::string> k_answers = {73 "The tallest mountain in the world is Mount Everest.",74 "Marie Curie was the first person to win two Nobel Prizes.",75 "Paper was invented in China.",76 "The heart is the organ responsible for pumping blood.",77 "Saturn is known for its prominent ring system.",78 "Christopher Nolan directed the movie 'Inception'.",79 "The freezing point of water in Fahrenheit is 32°F.",80 "The bowhead whale is known to have the longest lifespan among mammals.",81 "Mandarin Chinese has the most native speakers in the world.",82 "The capital city of Canada is Ottawa.",83 "Tim Berners-Lee is credited with inventing the World Wide Web.",84 "Mercury is the metal that is liquid at room temperature.",85 "An animal that eats both plants and meat is called an omnivore.",86 "'The Starry Night' was painted by Vincent van Gogh.",87 "Humans exhale carbon dioxide, which plants use in photosynthesis.",88 "World War II ended in 1945.",89 "Africa is the continent with the most countries.",90 "The novel 'Frankenstein' was written by Mary Shelley.",91 "DNA stands for Deoxyribonucleic Acid.",92 "The main ingredient in traditional Japanese miso soup is fermented soybean paste."93};94 95static std::vector<std::string> k_prompts = {96 "What is the meaning of life?",97 "Tell me an interesting fact about llamas.",98 "What is the best way to cook a steak?",99 "Are you familiar with the Special Theory of Relativity and can you explain it to me?",100 "Recommend some interesting books to read.",101 "What is the best way to learn a new language?",102 "How to get a job at Google?",103 "If you could have any superpower, what would it be?",104 "I want to learn how to play the piano. What would be the best way to do it?",105};106 107struct client {108 ~client() {109 if (smpl) {110 common_sampler_free(smpl);111 }112 }113 114 int32_t id = 0;115 116 llama_seq_id seq_id = -1;117 118 llama_token sampled;119 120 int64_t t_start_prompt;121 int64_t t_start_gen;122 123 int32_t n_past = 0;124 int32_t n_prompt = 0;125 int32_t n_decoded = 0;126 int32_t i_batch = -1;127 128 std::string input;129 std::string prompt;130 std::string response;131 132 struct common_sampler * smpl = nullptr;133};134 135static void print_date_time() {136 std::time_t current_time = std::time(nullptr);137 std::tm* local_time = std::localtime(¤t_time);138 char buffer[80];139 strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", local_time);140 141 LOG_INF("\n");142 LOG_INF("\033[35mrun parameters as of %s\033[0m\n", buffer);143 LOG_INF("\n");144}145 146// Define a split string function to ...147static std::vector<std::string> split_string(const std::string& input, char delimiter) {148 std::vector<std::string> tokens;149 std::istringstream stream(input);150 std::string token;151 while (std::getline(stream, token, delimiter)) {152 tokens.push_back(token);153 }154 return tokens;155}156 157int main(int argc, char ** argv) {158 std::setlocale(LC_NUMERIC, "C");159 160 std::mt19937 rng(1234);161 162 common_params params;163 164 params.n_predict = 128;165 params.n_junk = 1;166 167 common_init();168 169 if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_PARALLEL)) {170 return 1;171 }172 173 // number of simultaneous "clients" to simulate174 const int32_t n_clients = params.n_parallel;175 176 // dedicate one sequence to the system prompt177 params.n_parallel += 1;178 179 // requests to simulate180 const int32_t n_seq = params.n_sequences;181 182 // insert new requests as soon as the previous one is done183 const bool cont_batching = params.cont_batching;184 185 // is the system prompt shared in the cache186 const bool is_sp_shared = params.is_pp_shared;187 188 // extra text to insert in each client's prompt in order to make it larger189 const int32_t n_junk = std::max(1, params.n_junk);190 191 // signed seed, use negative values to indicate different seeds for the different clients192 const int32_t & sseed = params.sampling.seed;193 194 // init llama.cpp195 llama_backend_init();196 llama_numa_init(params.numa);197 198 // load the target model199 auto llama_init = common_init_from_params(params);200 201 auto * model = llama_init->model();202 auto * ctx = llama_init->context();203 204 auto * mem = llama_get_memory(ctx);205 206 const llama_vocab * vocab = llama_model_get_vocab(model);207 208 // load the prompts from an external file if there are any209 if (params.prompt.empty()) {210 LOG_INF("\033[32mNo new questions so proceed with build-in defaults.\033[0m\n");211 } else {212 // Output each line of the input params.prompts vector and copy to k_prompts213 int index = 0;214 LOG_INF("\033[32mNow printing the external prompt file %s\033[0m\n\n", params.prompt_file.c_str());215 216 std::vector<std::string> prompts = split_string(params.prompt, '\n');217 for (const auto& prompt : prompts) {218 k_prompts.resize(index + 1);219 k_prompts[index] = prompt;220 index++;221 LOG_INF("%3d prompt: %s\n", index, prompt.c_str());222 }223 }224 225 LOG_INF("\n\n");226 227 const int n_ctx = llama_n_ctx(ctx);228 229 if (sseed >= 0) {230 LOG_INF("%s: initializing all samplers with the same RNG seed: %d (use a negative seed to have different seeds)\n", __func__, sseed);231 } else {232 LOG_INF("%s: initializing samplers with different RNG seeds, starting from %d\n", __func__, sseed);233 }234 235 std::vector<client> clients(n_clients);236 for (size_t i = 0; i < clients.size(); ++i) {237 auto & client = clients[i];238 client.id = i;239 client.smpl = common_sampler_init(model, params.sampling);240 241 if (sseed < 0) {242 params.sampling.seed--;243 }244 }245 246 std::vector<llama_token> tokens_system;247 248 tokens_system = common_tokenize(ctx, k_system, true);249 const int32_t n_tokens_system = tokens_system.size();250 251 llama_seq_id g_seq_id = 0;252 253 // the max batch size is as large as the context to handle cases where we get very long input prompt from multiple254 // users. regardless of the size, the main loop will chunk the batch into a maximum of params.n_batch tokens at a time255 llama_batch batch = llama_batch_init(n_ctx, 0, 1);256 257 int32_t n_total_prompt = 0;258 int32_t n_total_gen = 0;259 int32_t n_cache_miss = 0;260 261 const auto t_main_start = ggml_time_us();262 263 LOG_INF("%s: Simulating parallel requests from clients:\n", __func__);264 LOG_INF("%s: n_parallel = %d, n_sequences = %d, cont_batching = %d, system tokens = %d\n", __func__, n_clients, n_seq, cont_batching, n_tokens_system);265 LOG_INF("\n");266 267 if (is_sp_shared) {268 LOG_INF("%s: Evaluating the system prompt ...\n", __func__);269 270 for (int32_t i = 0; i < n_tokens_system; ++i) {271 common_batch_add(batch, tokens_system[i], i, { 0 }, false);272 }273 274 if (llama_decode(ctx, batch) != 0) {275 LOG_ERR("%s: llama_decode() failed\n", __func__);276 return 1;277 }278 279 // assign the system KV cache to all parallel sequences280 for (int32_t i = 1; i <= n_clients; ++i) {281 llama_memory_seq_cp(mem, 0, i, -1, -1);282 }283 284 LOG_INF("\n");285 }286 287 LOG_INF("Processing requests ...\n\n");288 289 while (true) {290 common_batch_clear(batch);291 292 // decode any currently ongoing sequences293 for (auto & client : clients) {294 if (client.seq_id == -1) {295 continue;296 }297 298 client.i_batch = batch.n_tokens;299 300 common_batch_add(batch, client.sampled, client.n_past++, { client.id + 1 }, true);301 302 client.n_decoded += 1;303 }304 305 if (batch.n_tokens == 0) {306 // all sequences have ended - clear the entire KV cache307 for (int i = 1; i <= n_clients; ++i) {308 llama_memory_seq_rm(mem, i, -1, -1);309 // but keep the system prompt310 llama_memory_seq_cp(mem, 0, i, -1, -1);311 }312 313 LOG_INF("%s: clearing the KV cache\n", __func__);314 }315 316 // insert new sequences for decoding317 if (cont_batching || batch.n_tokens == 0) {318 for (auto & client : clients) {319 if (client.seq_id == -1 && g_seq_id < n_seq) {320 client.seq_id = g_seq_id;321 322 client.t_start_prompt = ggml_time_us();323 client.t_start_gen = 0;324 325 client.input = k_prompts[rng() % k_prompts.size()];326 client.response = "";327 328 // construct the prompt:329 // [system prompt] + [junk] + [user prompt]330 client.n_past = 0;331 client.prompt = "";332 if (is_sp_shared) {333 client.n_past = n_tokens_system;334 } else {335 client.prompt += k_system;336 }337 338 const int n_junk_cur = rng() % n_junk;339 340 for (int i = 0; i < n_junk_cur; ++i) {341 const int r = rng() % k_questions.size();342 client.prompt += "User:\n" + k_questions[r] + "\nAssistant:\n " + k_answers[r] + "\n";343 }344 client.prompt += "User:\n" + client.input + "\nAssistant:\n";345 346 common_sampler_reset(client.smpl);347 348 // do not prepend BOS because we have a system prompt!349 std::vector<llama_token> tokens_prompt;350 tokens_prompt = common_tokenize(ctx, client.prompt, false);351 352 for (size_t i = 0; i < tokens_prompt.size(); ++i) {353 common_batch_add(batch, tokens_prompt[i], client.n_past++, { client.id + 1 }, false);354 }355 356 // extract the logits only for the last token357 if (batch.n_tokens > 0) {358 batch.logits[batch.n_tokens - 1] = true;359 }360 361 client.n_prompt = tokens_prompt.size();362 client.n_decoded = 0;363 client.i_batch = batch.n_tokens - 1;364 365 LOG_INF("\033[31mClient %3d, seq %4d, junk = %4d, prompt = %d, started decoding ...\033[0m\n", client.id, client.seq_id, n_junk_cur, client.n_prompt);366 367 g_seq_id += 1;368 369 // insert new requests one-by-one370 //if (cont_batching) {371 // break;372 //}373 }374 }375 }376 377 if (batch.n_tokens == 0) {378 break;379 }380 381 // process in chunks of params.n_batch382 int32_t n_batch = params.n_batch;383 384 int32_t i_next = 0;385 386 for (int32_t i = 0; i < batch.n_tokens; i = i_next) {387 // experiment: process in powers of 2388 //if (i + n_batch > (int32_t) batch.n_tokens && n_batch > 32) {389 // n_batch /= 2;390 // i -= n_batch;391 // continue;392 //}393 394 const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);395 396 llama_batch batch_view = {397 n_tokens,398 batch.token + i,399 nullptr,400 batch.pos + i,401 batch.n_seq_id + i,402 batch.seq_id + i,403 batch.logits + i,404 };405 406 const int ret = llama_decode(ctx, batch_view);407 if (ret != 0) {408 if (n_batch == 1 || ret < 0) {409 // if you get here, it means the KV cache is full - try increasing it via the context size410 LOG_ERR("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret);411 return 1;412 }413 414 LOG_WRN("%s : failed to decode the batch, retrying with n_batch = %d\n", __func__, n_batch / 2);415 416 n_cache_miss += 1;417 418 // retry with half the batch size to try to find a free slot in the KV cache419 n_batch /= 2;420 421 continue;422 }423 424 LOG_DBG("%s : decoded batch of %d tokens\n", __func__, n_tokens);425 426 // move the head of the batch forward with the number of tokens we just processed427 i_next = i + n_tokens;428 429 // on successful decode, restore the original batch size430 n_batch = params.n_batch;431 432 for (auto & client : clients) {433 if (client.i_batch < (int) i || client.i_batch >= (int) (i + n_tokens)) {434 continue;435 }436 437 //printf("client %d, seq %d, token %d, pos %d, batch %d\n",438 // client.id, client.seq_id, client.sampled, client.n_decoded, client.i_batch);439 440 const llama_token id = common_sampler_sample(client.smpl, ctx, client.i_batch - i);441 442 common_sampler_accept(client.smpl, id, true);443 444 if (client.n_decoded == 1) {445 // start measuring generation time after the first token to make sure all concurrent clients446 // have their prompt already processed447 client.t_start_gen = ggml_time_us();448 }449 450 const std::string token_str = common_token_to_piece(ctx, id);451 452 client.response += token_str;453 client.sampled = id;454 455 //printf("client %d, seq %d, token %d, pos %d, batch %d: %s\n",456 // client.id, client.seq_id, id, client.n_decoded, client.i_batch, token_str.c_str());457 458 if (client.n_decoded > 2 &&459 (llama_vocab_is_eog(vocab, id) ||460 (params.n_predict > 0 && client.n_decoded >= params.n_predict) ||461 client.response.find("User:") != std::string::npos)) {462 // basic reverse prompt463 const size_t pos = client.response.find("User:");464 if (pos != std::string::npos) {465 client.response = client.response.substr(0, pos);466 }467 468 // delete only the generated part of the sequence, i.e. keep the system prompt in the cache469 llama_memory_seq_rm(mem, client.id + 1, -1, -1);470 llama_memory_seq_cp(mem, 0, client.id + 1, -1, -1);471 472 const auto t_main_end = ggml_time_us();473 474 LOG_INF("\033[31mClient %3d, seq %3d/%3d, prompt %4d t, response %4d t, time %5.2f s, speed %5.2f t/s, cache miss %d \033[0m \n\nInput: %s\n\033[35mResponse: %s\033[0m\n\n",475 client.id, client.seq_id, n_seq, client.n_prompt, client.n_decoded,476 (t_main_end - client.t_start_prompt) / 1e6,477 (double) (client.n_prompt + client.n_decoded) / (t_main_end - client.t_start_prompt) * 1e6,478 n_cache_miss,479 ::trim(client.input).c_str(),480 ::trim(client.response).c_str());481 482 n_total_prompt += client.n_prompt;483 n_total_gen += client.n_decoded;484 485 client.seq_id = -1;486 }487 488 client.i_batch = -1;489 }490 }491 }492 493 const auto t_main_end = ggml_time_us();494 495 print_date_time();496 497 LOG_INF("%s: n_parallel = %d, n_sequences = %d, cont_batching = %d, system tokens = %d\n", __func__, n_clients, n_seq, cont_batching, n_tokens_system);498 if (params.prompt_file.empty()) {499 params.prompt_file = "used built-in defaults";500 }501 LOG_INF("External prompt file: \033[32m%s\033[0m\n", params.prompt_file.c_str());502 LOG_INF("Model and path used: \033[32m%s\033[0m\n\n", params.model.path.c_str());503 504 LOG_INF("Total prompt tokens: %6d, speed: %5.2f t/s\n", n_total_prompt, (double) (n_total_prompt ) / (t_main_end - t_main_start) * 1e6);505 LOG_INF("Total gen tokens: %6d, speed: %5.2f t/s\n", n_total_gen, (double) (n_total_gen ) / (t_main_end - t_main_start) * 1e6);506 LOG_INF("Total speed (AVG): %6s speed: %5.2f t/s\n", "", (double) (n_total_prompt + n_total_gen) / (t_main_end - t_main_start) * 1e6);507 LOG_INF("Cache misses: %6d\n", n_cache_miss);508 509 LOG_INF("\n");510 511 // TODO: print sampling/grammar timings for all clients512 llama_perf_context_print(ctx);513 514 llama_batch_free(batch);515 516 llama_backend_free();517 518 LOG("\n\n");519 520 return 0;521}522