Felipe97/llama-cpp-compiled
01.1k
1#include "arg.h"2#include "common.h"3#include "download.h"4#include "llama.h"5#include "speculative.h"6 7#include <cmath>8#include <limits>9#include <string>10#include <vector>11#include <sstream>12#include <unordered_set>13 14#undef NDEBUG15#include <cassert>16 17static void test(void) {18 common_params params;19 20 auto assert_output_limits = [](int32_t n_batch, int32_t n_parallel, int32_t n_draft,21 int32_t total, int32_t per_seq) {22 const auto limits = common_speculative_get_output_limits(n_batch, n_parallel, n_draft);23 assert(limits.total == total);24 assert(limits.per_seq == per_seq);25 };26 27 assert_output_limits(16, 2, 3, 8, 4);28 assert_output_limits(16, 2, -1, 2, 1);29 assert_output_limits( 6, 2, 3, 6, 4);30 assert_output_limits( 2, 1, 3, 2, 2);31 assert_output_limits(32 std::numeric_limits<int32_t>::max(),33 std::numeric_limits<int32_t>::max(),34 std::numeric_limits<int32_t>::max(),35 std::numeric_limits<int32_t>::max(),36 std::numeric_limits<int32_t>::max());37 38 {39 common_params_speculative spec;40 spec.synth_len = 3.4;41 42 auto assert_invalid = [](const common_params_speculative & value, int32_t n_max) {43 try {44 common_speculative_synth_rates_resolve(&value, n_max);45 assert(false);46 } catch (const std::invalid_argument &) {47 }48 };49 50 const auto rates = common_speculative_synth_rates_resolve(&spec, 4);51 assert(rates.size() == 4);52 assert(std::abs(rates[0] - 0.80581) < 1e-5);53 assert(std::abs(rates[1] - 0.64933) < 1e-5);54 assert(std::abs(rates[2] - 0.52323) < 1e-5);55 assert(std::abs(rates[3] - 0.42163) < 1e-5);56 assert(std::abs(1.0 + rates[0] + rates[1] + rates[2] + rates[3] - 3.4) < 1e-8);57 58 spec.synth_len = 1.0;59 assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({0.0, 0.0, 0.0, 0.0}));60 61 spec.synth_len = 5.0;62 assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({1.0, 1.0, 1.0, 1.0}));63 64 spec.synth_len = 5.1;65 assert_invalid(spec, 4);66 67 spec.synth_len = std::numeric_limits<double>::quiet_NaN();68 assert_invalid(spec, 4);69 70 spec.synth_len = 0.0;71 assert_invalid(spec, 4);72 73 spec.synth_len = -1.0;74 spec.synth_rates = {0.8, 0.6, 0.4};75 assert_invalid(spec, 4);76 77 spec.synth_rates = {0.8, 0.6, 0.4, 0.2};78 assert(common_speculative_synth_rates_resolve(&spec, 4) == spec.synth_rates);79 80 spec.synth_rates = {0.8, 0.9, 0.4, 0.2};81 assert_invalid(spec, 4);82 83 spec.synth_rates = {0.8, std::numeric_limits<double>::quiet_NaN(), 0.4, 0.2};84 assert_invalid(spec, 4);85 86 spec.synth_rates = {0.8, 0.6, 0.4, -0.2};87 assert_invalid(spec, 4);88 89 spec.synth_rates = {0.8, 0.6, 0.4, 0.2};90 spec.synth_len = 3.0;91 assert_invalid(spec, 4);92 }93 94 {95 common_params base;96 base.n_parallel = 4;97 base.n_outputs_max_per_seq = 8;98 99 const auto draft = common_base_params_to_speculative(base);100 assert(draft.n_outputs_max == 4);101 assert(draft.n_outputs_max_per_seq == 1);102 }103 104 printf("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n");105 for (int ex = 0; ex < LLAMA_EXAMPLE_COUNT; ex++) {106 try {107 auto ctx_arg = common_params_parser_init(params, (enum llama_example)ex);108 common_params_add_preset_options(ctx_arg.options);109 std::unordered_set<std::string> seen_args;110 std::unordered_set<std::string> seen_env_vars;111 for (const auto & opt : ctx_arg.options) {112 // check for args duplications113 for (const auto & arg : opt.get_args()) {114 if (seen_args.find(arg) == seen_args.end()) {115 seen_args.insert(arg);116 } else {117 fprintf(stderr, "test-arg-parser: found different handlers for the same argument: %s", arg.c_str());118 exit(1);119 }120 }121 // check for env var duplications122 for (const auto & env : opt.get_env()) {123 if (seen_env_vars.find(env) == seen_env_vars.end()) {124 seen_env_vars.insert(env);125 } else {126 fprintf(stderr, "test-arg-parser: found different handlers for the same env var: %s", env.c_str());127 exit(1);128 }129 }130 131 // exclude spec args from this check132 // ref: https://github.com/ggml-org/llama.cpp/pull/22397133 const bool skip = opt.is_spec;134 135 // ensure shorter argument precedes longer argument136 if (!skip && opt.args.size() > 1) {137 const std::string first(opt.args.front());138 const std::string last(opt.args.back());139 140 if (first.length() > last.length()) {141 fprintf(stderr, "test-arg-parser: shorter argument should come before longer one: %s, %s\n",142 first.c_str(), last.c_str());143 assert(false);144 }145 }146 147 // same check for negated arguments148 if (opt.args_neg.size() > 1) {149 const std::string first(opt.args_neg.front());150 const std::string last(opt.args_neg.back());151 152 if (first.length() > last.length()) {153 fprintf(stderr, "test-arg-parser: shorter negated argument should come before longer one: %s, %s\n",154 first.c_str(), last.c_str());155 assert(false);156 }157 }158 }159 } catch (std::exception & e) {160 printf("%s\n", e.what());161 assert(false);162 }163 }164 165 auto list_str_to_char = [](std::vector<std::string> & argv) -> std::vector<char *> {166 std::vector<char *> res;167 for (auto & arg : argv) {168 res.push_back(const_cast<char *>(arg.data()));169 }170 return res;171 };172 173 std::vector<std::string> argv;174 175 printf("test-arg-parser: test invalid usage\n\n");176 177 // missing value178 argv = {"binary_name", "-m"};179 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));180 181 // wrong value (int)182 argv = {"binary_name", "-ngl", "hello"};183 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));184 185 // wrong value (enum)186 argv = {"binary_name", "-sm", "hello"};187 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));188 189 {190 common_params penalty_params;191 assert(penalty_params.sampling.penalty_last_n == 64);192 assert(penalty_params.sampling.dry_penalty_last_n == 64);193 194 argv = {"binary_name", "--repeat-last-n", "-1"};195 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));196 197 argv = {"binary_name", "--dry-penalty-last-n", "-1"};198 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));199 200 argv = {"binary_name", "--repeat-penalty", "0"};201 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));202 203 argv = {"binary_name", "--repeat-penalty", "-1"};204 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));205 206 argv = {"binary_name", "--repeat-penalty", "nan"};207 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));208 209 argv = {"binary_name", "--repeat-penalty", "inf"};210 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));211 212 argv = {"binary_name", "--repeat-penalty", "-inf"};213 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));214 215 const char * penalty_options[] = {"--frequency-penalty", "--presence-penalty"};216 const char * nonfinite_values[] = {"nan", "inf", "-inf"};217 for (const char * option : penalty_options) {218 for (const char * value : nonfinite_values) {219 argv = {"binary_name", option, value};220 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));221 }222 }223 }224 225 // non-existence arg in specific example (--draft cannot be used outside llama-speculative)226 argv = {"binary_name", "--draft", "123"};227 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));228 229 argv = {"binary_name", "-lm", "hello"};230 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));231 232 printf("test-arg-parser: test valid usage\n\n");233 234 argv = {"binary_name", "-m", "model_file.gguf"};235 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));236 assert(params.model.path == "model_file.gguf");237 238 argv = {"binary_name", "-t", "1234"};239 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));240 assert(params.cpuparams.n_threads == 1234);241 242 argv = {"binary_name", "--verbose"};243 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));244 assert(params.verbosity > 1);245 246 argv = {"binary_name", "-m", "abc.gguf", "--predict", "6789", "--batch-size", "9090"};247 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));248 assert(params.model.path == "abc.gguf");249 assert(params.n_predict == 6789);250 assert(params.n_batch == 9090);251 252 // --draft cannot be used outside llama-speculative253 argv = {"binary_name", "--spec-draft-n-max", "123"};254 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));255 assert(params.speculative.draft.n_max == 123);256 257 {258 common_params synth_params;259 argv = {"binary_name", "--spec-synth-len", "3.4"};260 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));261 assert(synth_params.speculative.synth_len == 3.4);262 }263 264 {265 common_params synth_params;266 argv = {"binary_name", "--spec-synth-rates", "0.8,0.6,0.2"};267 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));268 assert(synth_params.speculative.synth_rates == std::vector<double>({0.8, 0.6, 0.2}));269 }270 271 {272 common_params synth_params;273 argv = {"binary_name", "--spec-synth-len", "3.4x"};274 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));275 }276 277 argv = {"binary_name", "-lm", "none"};278 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));279 assert(params.load_mode == LLAMA_LOAD_MODE_NONE);280 281 argv = {"binary_name", "-lm", "mmap"};282 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));283 assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);284 285 argv = {"binary_name", "-lm", "mlock"};286 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));287 assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);288 289 argv = {"binary_name", "-lm", "mmap+mlock"};290 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));291 assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);292 293 argv = {"binary_name", "-lm", "dio"};294 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));295 assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);296 297 // multi-value args (CSV)298 argv = {"binary_name", "--lora", "file1.gguf,\"file2,2.gguf\",\"file3\"\"3\"\".gguf\",file4\".gguf"};299 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));300 assert(params.lora_adapters.size() == 4);301 assert(params.lora_adapters[0].path == "file1.gguf");302 assert(params.lora_adapters[1].path == "file2,2.gguf");303 assert(params.lora_adapters[2].path == "file3\"3\".gguf");304 assert(params.lora_adapters[3].path == "file4\".gguf");305 306// skip this part on windows, because setenv is not supported307#ifdef _WIN32308 printf("test-arg-parser: skip on windows build\n");309#else310 printf("test-arg-parser: test environment variables (valid + invalid usages)\n\n");311 312 setenv("LLAMA_ARG_THREADS", "blah", true);313 argv = {"binary_name"};314 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));315 316 setenv("LLAMA_ARG_MODEL", "blah.gguf", true);317 setenv("LLAMA_ARG_THREADS", "1010", true);318 argv = {"binary_name"};319 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));320 assert(params.model.path == "blah.gguf");321 assert(params.cpuparams.n_threads == 1010);322 323 setenv("LLAMA_ARG_LOAD_MODE", "blah", true);324 argv = {"binary_name"};325 assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));326 327 setenv("LLAMA_ARG_LOAD_MODE", "mmap", true);328 argv = {"binary_name"};329 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));330 assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);331 332 setenv("LLAMA_ARG_LOAD_MODE", "mlock", true);333 argv = {"binary_name"};334 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));335 assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);336 337 setenv("LLAMA_ARG_LOAD_MODE", "mmap+mlock", true);338 argv = {"binary_name"};339 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));340 assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);341 342 setenv("LLAMA_ARG_LOAD_MODE", "dio", true);343 argv = {"binary_name"};344 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));345 assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);346 347 printf("test-arg-parser: test negated environment variables\n\n");348 349 setenv("LLAMA_ARG_LOAD_MODE", "none", true);350 setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy format351 argv = {"binary_name"};352 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));353 assert(params.load_mode == LLAMA_LOAD_MODE_NONE);354 assert(params.no_perf == true);355 356 printf("test-arg-parser: test environment variables being overwritten\n\n");357 358 setenv("LLAMA_ARG_MODEL", "blah.gguf", true);359 setenv("LLAMA_ARG_THREADS", "1010", true);360 argv = {"binary_name", "-m", "overwritten.gguf"};361 assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));362 assert(params.model.path == "overwritten.gguf");363 assert(params.cpuparams.n_threads == 1010);364#endif // _WIN32365 366 printf("test-arg-parser: test download functions\n\n");367 const char * GOOD_URL = "http://ggml.ai/";368 const char * BAD_URL = "http://ggml.ai/404";369 370 {371 printf("test-arg-parser: test good URL\n\n");372 auto res = common_remote_get_content(GOOD_URL, {});373 assert(res.first == 200);374 assert(res.second.size() > 0);375 std::string str(res.second.data(), res.second.size());376 assert(str.find("llama.cpp") != std::string::npos);377 }378 379 {380 printf("test-arg-parser: test bad URL\n\n");381 auto res = common_remote_get_content(BAD_URL, {});382 assert(res.first == 404);383 }384 385 {386 printf("test-arg-parser: test max size error\n");387 common_remote_params params;388 params.max_size = 1;389 try {390 common_remote_get_content(GOOD_URL, params);391 assert(false && "it should throw an error");392 } catch (std::exception & e) {393 printf(" expected error: %s\n\n", e.what());394 }395 }396 397 printf("test-arg-parser: all tests OK\n\n");398}399 400int main(void) {401 try {402 test();403 } catch (std::exception & e) {404 fprintf(stderr, "test-arg-parser: exception: %s\n", e.what());405 return 1;406 }407 return 0;408}409