Felipe97/llama-cpp-compiled
01.1k
1#include "llama.h"2 3#include "build-info.h"4#include "common.h"5#include "imatrix-loader.h"6 7#include "gguf.h"8 9#include <algorithm>10#include <cctype>11#include <clocale>12#include <cmath>13#include <cstdio>14#include <cstring>15#include <vector>16#include <string>17#include <unordered_map>18#include <fstream>19#include <filesystem>20 21// result of parsing --tensor-type option22// changes to this struct must also be reflected in src/llama-quant.cpp23struct tensor_type_option {24 std::string name;25 ggml_type type = GGML_TYPE_COUNT;26};27 28struct quant_option {29 std::string name;30 llama_ftype ftype;31 std::string desc;32};33 34static const std::vector<quant_option> QUANT_OPTIONS = {35 { "Q1_0", LLAMA_FTYPE_MOSTLY_Q1_0, " 1.125 bpw quantization", },36 { "Q2_0", LLAMA_FTYPE_MOSTLY_Q2_0, " 2.25 bpw quantization (group 64)", },37 { "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0, " 4.34G, +0.4685 ppl @ Llama-3-8B", },38 { "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 4.78G, +0.4511 ppl @ Llama-3-8B", },39 { "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE", },40 { "Q5_0", LLAMA_FTYPE_MOSTLY_Q5_0, " 5.21G, +0.1316 ppl @ Llama-3-8B", },41 { "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 5.65G, +0.1062 ppl @ Llama-3-8B", },42 { "IQ2_XXS", LLAMA_FTYPE_MOSTLY_IQ2_XXS, " 2.06 bpw quantization", },43 { "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", },44 { "IQ2_S", LLAMA_FTYPE_MOSTLY_IQ2_S, " 2.5 bpw quantization", },45 { "IQ2_M", LLAMA_FTYPE_MOSTLY_IQ2_M, " 2.7 bpw quantization", },46 { "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S, " 1.56 bpw quantization", },47 { "IQ1_M", LLAMA_FTYPE_MOSTLY_IQ1_M, " 1.75 bpw quantization", },48 { "TQ1_0", LLAMA_FTYPE_MOSTLY_TQ1_0, " 1.69 bpw ternarization", },49 { "TQ2_0", LLAMA_FTYPE_MOSTLY_TQ2_0, " 2.06 bpw ternarization", },50 { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.96G, +3.5199 ppl @ Llama-3-8B", },51 { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.96G, +3.1836 ppl @ Llama-3-8B", },52 { "IQ3_XXS", LLAMA_FTYPE_MOSTLY_IQ3_XXS, " 3.06 bpw quantization", },53 { "IQ3_S", LLAMA_FTYPE_MOSTLY_IQ3_S, " 3.44 bpw quantization", },54 { "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M, " 3.66 bpw quantization mix", },55 { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" },56 { "IQ3_XS", LLAMA_FTYPE_MOSTLY_IQ3_XS, " 3.3 bpw quantization", },57 { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 3.41G, +1.6321 ppl @ Llama-3-8B", },58 { "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.74G, +0.6569 ppl @ Llama-3-8B", },59 { "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 4.03G, +0.5562 ppl @ Llama-3-8B", },60 { "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL, " 4.50 bpw non-linear quantization", },61 { "IQ4_XS", LLAMA_FTYPE_MOSTLY_IQ4_XS, " 4.25 bpw non-linear quantization", },62 { "Q4_K", LLAMA_FTYPE_MOSTLY_Q4_K_M, "alias for Q4_K_M", },63 { "Q4_K_S", LLAMA_FTYPE_MOSTLY_Q4_K_S, " 4.37G, +0.2689 ppl @ Llama-3-8B", },64 { "Q4_K_M", LLAMA_FTYPE_MOSTLY_Q4_K_M, " 4.58G, +0.1754 ppl @ Llama-3-8B", },65 { "Q5_K", LLAMA_FTYPE_MOSTLY_Q5_K_M, "alias for Q5_K_M", },66 { "Q5_K_S", LLAMA_FTYPE_MOSTLY_Q5_K_S, " 5.21G, +0.1049 ppl @ Llama-3-8B", },67 { "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 5.33G, +0.0569 ppl @ Llama-3-8B", },68 { "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 6.14G, +0.0217 ppl @ Llama-3-8B", },69 { "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0, " 7.96G, +0.0026 ppl @ Llama-3-8B", },70 { "F16", LLAMA_FTYPE_MOSTLY_F16, "14.00G, +0.0020 ppl @ Mistral-7B", },71 { "BF16", LLAMA_FTYPE_MOSTLY_BF16, "14.00G, -0.0050 ppl @ Mistral-7B", },72 { "F32", LLAMA_FTYPE_ALL_F32, "26.00G @ 7B", },73 // Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.74 { "COPY", LLAMA_FTYPE_ALL_F32, "only copy tensors, no quantizing", },75};76 77static const char * const LLM_KV_QUANTIZE_IMATRIX_FILE = "quantize.imatrix.file";78static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrix.dataset";79static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";80static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";81 82static bool striequals(const char * a, const char * b) {83 while (*a && *b) {84 if (std::tolower(*a) != std::tolower(*b)) {85 return false;86 }87 a++; b++;88 }89 return *a == *b;90}91 92static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std::string & ftype_str_out) {93 std::string ftype_str;94 95 for (auto ch : ftype_str_in) {96 ftype_str.push_back(std::toupper(ch));97 }98 for (const auto & it : QUANT_OPTIONS) {99 if (striequals(it.name.c_str(), ftype_str.c_str())) {100 ftype = it.ftype;101 ftype_str_out = it.name;102 return true;103 }104 }105 try {106 int ftype_int = std::stoi(ftype_str);107 for (const auto & it : QUANT_OPTIONS) {108 if (it.ftype == ftype_int) {109 ftype = it.ftype;110 ftype_str_out = it.name;111 return true;112 }113 }114 }115 catch (...) {116 // stoi failed117 }118 return false;119}120 121[[noreturn]]122static void usage(const char * executable) {123 printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);124 printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");125 printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run] [--max-buffer-size]\n");126 printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");127 printf(" --allow-requantize\n");128 printf(" allow requantizing tensors that have already been quantized\n");129 printf(" WARNING: this can severely reduce quality compared to quantizing\n");130 printf(" from 16bit or 32bit!\n");131 printf(" --leave-output-tensor\n");132 printf(" leave output.weight un(re)quantized\n");133 printf(" increases model size but may also increase quality, especially when requantizing\n");134 printf(" --pure\n");135 printf(" disable k-quant mixtures and quantize all tensors to the same type\n");136 printf(" --imatrix file_name\n");137 printf(" use data in file_name as importance matrix for quant optimizations\n");138 printf(" --include-weights tensor_name\n");139 printf(" use importance matrix for this/these tensor(s)\n");140 printf(" --exclude-weights tensor_name\n");141 printf(" do not use importance matrix for this/these tensor(s)\n");142 printf(" --output-tensor-type ggml_type\n");143 printf(" use this ggml_type for the output.weight tensor\n");144 printf(" --token-embedding-type ggml_type\n");145 printf(" use this ggml_type for the token embeddings tensor\n");146 printf(" --tensor-type tensor_name=ggml_type\n");147 printf(" quantize this tensor to this ggml_type\n");148 printf(" this is an advanced option to selectively quantize tensors. may be specified multiple times.\n");149 printf(" example: --tensor-type attn_q=q8_0\n");150 printf(" --tensor-type-file tensor_types.txt\n");151 printf(" list of tensors to quantize to a specific ggml_type\n");152 printf(" this is an advanced option to selectively quantize a long list of tensors.\n");153 printf(" the file should use the same format as above, separated by spaces or newlines.\n");154 printf(" --prune-layers L0,L1,L2...\n");155 printf(" comma-separated list of layer numbers to prune from the model\n");156 printf(" WARNING: this is an advanced option, use with care.\n");157 printf(" --keep-split\n");158 printf(" generate quantized model in the same shards as input\n");159 printf(" --override-kv KEY=TYPE:VALUE\n");160 printf(" override model metadata by key in the quantized model. may be specified multiple times.\n");161 printf(" WARNING: this is an advanced option, use with care.\n");162 printf(" --dry-run\n");163 printf(" calculate and show the final quantization size without performing quantization\n");164 printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n");165 printf(" --max-buffer-size MiB\n");166 printf(" max amount of tensor rows kept in memory while quantizing one tensor (default: 8192)\n");167 printf(" lower it to quantize models with very large tensors on a machine with little RAM\n\n");168 printf("note: --include-weights and --exclude-weights cannot be used together\n\n");169 printf("-----------------------------------------------------------------------------\n");170 printf(" allowed quantization types\n");171 printf("-----------------------------------------------------------------------------\n\n");172 for (const auto & it : QUANT_OPTIONS) {173 if (it.name != "COPY") {174 printf(" %2d or ", it.ftype);175 } else {176 printf(" ");177 }178 printf("%-7s : %s\n", it.name.c_str(), it.desc.c_str());179 }180 exit(1);181}182 183static int load_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {184 common_imatrix loaded;185 if (!common_imatrix_load(imatrix_file, loaded)) {186 fprintf(stderr, "%s: failed to load imatrix from '%s'\n", __func__, imatrix_file.c_str());187 exit(1);188 }189 190 if (!loaded.is_legacy && !loaded.has_metadata) {191 fprintf(stderr, "%s: missing imatrix metadata in file %s\n", __func__, imatrix_file.c_str());192 exit(1);193 }194 195 for (const auto & [name, entry] : loaded.entries) {196 auto & e = imatrix_data[name];197 e.resize(entry.sums.size());198 199 if (!loaded.is_legacy) {200 // GGUF format: normalize by per-expert counts201 const int64_t ncounts = entry.counts.size();202 const int64_t ne0 = (int64_t) entry.sums.size() / ncounts;203 204 for (int64_t j = 0; j < ncounts; ++j) {205 const float count = (float) entry.counts[j];206 if (count > 0.0f) {207 for (int64_t i = 0; i < ne0; ++i) {208 e[j*ne0 + i] = entry.sums[j*ne0 + i] / count;209 }210 } else {211 for (int64_t i = 0; i < ne0; ++i) {212 e[j*ne0 + i] = 1;213 }214 }215 }216 217 if (getenv("LLAMA_TRACE")) {218 float max_count = 0.0f;219 for (int64_t j = 0; j < ncounts; ++j) {220 const float count = (float) entry.counts[j];221 if (count > max_count) {222 max_count = count;223 }224 }225 printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n",226 __func__, int(e.size()), int(max_count), int(max_count / loaded.chunk_size), name.c_str());227 }228 } else {229 // Legacy format: sums contain (raw/count)*ncall, divide by ncall230 const int64_t ncall = entry.counts.empty() ? 0 : entry.counts[0];231 if (ncall > 0) {232 for (size_t i = 0; i < entry.sums.size(); ++i) {233 e[i] = entry.sums[i] / ncall;234 }235 } else {236 for (size_t i = 0; i < entry.sums.size(); ++i) {237 e[i] = entry.sums[i];238 }239 }240 241 if (getenv("LLAMA_TRACE")) {242 printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n",243 __func__, int(e.size()), int(ncall), name.c_str());244 }245 }246 }247 248 imatrix_datasets = std::move(loaded.datasets);249 250 if (!imatrix_datasets.empty()) {251 printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());252 for (size_t i = 1; i < imatrix_datasets.size(); ++i) {253 printf(", '%s'", imatrix_datasets[i].c_str());254 }255 printf("]\n");256 }257 258 printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), loaded.chunk_count);259 260 return loaded.chunk_count;261}262 263static int prepare_imatrix(const std::string & imatrix_file,264 std::vector<std::string> & imatrix_dataset,265 const std::vector<std::string> & included_weights,266 const std::vector<std::string> & excluded_weights,267 std::unordered_map<std::string, std::vector<float>> & imatrix_data) {268 int m_last_call = -1;269 if (!imatrix_file.empty()) {270 m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data);271 }272 if (imatrix_data.empty()) {273 return m_last_call;274 }275 if (!excluded_weights.empty()) {276 for (const auto & name : excluded_weights) {277 for (auto it = imatrix_data.begin(); it != imatrix_data.end();) {278 auto pos = it->first.find(name);279 if (pos != std::string::npos) {280 it = imatrix_data.erase(it);281 } else {282 ++it;283 }284 }285 }286 }287 if (!included_weights.empty()) {288 std::unordered_map<std::string, std::vector<float>> tmp;289 for (const auto & name : included_weights) {290 for (auto & e : imatrix_data) {291 auto pos = e.first.find(name);292 if (pos != std::string::npos) {293 tmp.emplace(std::move(e));294 }295 }296 }297 imatrix_data = std::move(tmp);298 }299 if (!imatrix_data.empty()) {300 printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size()));301 }302 return m_last_call;303}304 305static ggml_type parse_ggml_type(const char * arg) {306 for (int i = 0; i < GGML_TYPE_COUNT; ++i) {307 auto type = (ggml_type)i;308 const auto * name = ggml_type_name(type);309 if (name && striequals(name, arg)) {310 return type;311 }312 }313 fprintf(stderr, "\n%s: invalid ggml_type '%s'\n\n", __func__, arg);314 return GGML_TYPE_COUNT;315}316 317static bool parse_tensor_type(const char * data, std::vector<tensor_type_option> & tensor_type) {318 const char * sep = strchr(data, '=');319 if (sep == nullptr) {320 printf("\n%s: malformed tensor type '%s'\n\n", __func__, data);321 return false;322 }323 324 const size_t tn_len = sep - data;325 if (tn_len == 0) {326 printf("\n%s: missing tensor name\n\n", __func__);327 return false;328 }329 if (const size_t qt_len = strlen(sep); qt_len == 1) {330 printf("\n%s: missing quantization type\n\n", __func__);331 return false;332 }333 334 std::string tn(data, tn_len);335 std::transform(tn.begin(), tn.end(), tn.begin(), tolower);336 sep++;337 tensor_type_option tensor_type_opt;338 tensor_type_opt.name = tn;339 tensor_type_opt.type = parse_ggml_type(sep);340 tensor_type.emplace_back(std::move(tensor_type_opt));341 if (tensor_type_opt.type == GGML_TYPE_COUNT) {342 printf("\n%s: invalid quantization type '%s'\n\n", __func__, sep);343 return false;344 }345 346 return true;347}348 349static bool parse_tensor_type_file(const char * filename, std::vector<tensor_type_option> & tensor_type) {350 std::ifstream file(filename);351 if (!file) {352 printf("\n%s: failed to open file '%s': %s\n\n", __func__, filename, std::strerror(errno));353 return false;354 }355 356 std::string arg;357 while (file >> arg) {358 if (!parse_tensor_type(arg.c_str(), tensor_type)) {359 return false;360 }361 }362 363 return true;364}365 366static bool parse_layer_prune(const char * data, std::vector<int> & prune_layers) {367 if (!data) {368 printf("\n%s: no layer pruning ids provided\n\n", __func__);369 return false;370 }371 372 const auto block_ids = string_split<std::string>(data, ',');373 for (const auto & block_id : block_ids) {374 int id;375 try {376 id = std::stoi(block_id);377 } catch (...) {378 id = -1;379 }380 if (id < 0) {381 printf("\n%s: invalid layer id '%s'\n\n", __func__, block_id.c_str());382 return false;383 }384 prune_layers.emplace_back(id);385 }386 387 sort(prune_layers.begin(), prune_layers.end());388 prune_layers.erase(std::unique(prune_layers.begin(), prune_layers.end()), prune_layers.end());389 return true;390}391 392// satisfies -Wmissing-declarations393int llama_quantize(int argc, char ** argv);394 395int llama_quantize(int argc, char ** argv) {396 std::setlocale(LC_NUMERIC, "C");397 if (argc < 3) {398 usage(argv[0]);399 }400 401 llama_model_quantize_params params = llama_model_quantize_default_params();402 403 int arg_idx = 1;404 std::string imatrix_file;405 std::vector<std::string> included_weights, excluded_weights;406 std::vector<llama_model_kv_override> kv_overrides;407 std::vector<tensor_type_option> tensor_type_opts;408 std::vector<int> prune_layers;409 410 for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {411 if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) {412 params.quantize_output_tensor = false;413 } else if (strcmp(argv[arg_idx], "--output-tensor-type") == 0) {414 if (arg_idx < argc-1) {415 params.output_tensor_type = parse_ggml_type(argv[++arg_idx]);416 if (params.output_tensor_type == GGML_TYPE_COUNT) {417 usage(argv[0]);418 }419 } else {420 usage(argv[0]);421 }422 } else if (strcmp(argv[arg_idx], "--token-embedding-type") == 0) {423 if (arg_idx < argc-1) {424 params.token_embedding_type = parse_ggml_type(argv[++arg_idx]);425 if (params.token_embedding_type == GGML_TYPE_COUNT) {426 usage(argv[0]);427 }428 } else {429 usage(argv[0]);430 }431 } else if (strcmp(argv[arg_idx], "--tensor-type") == 0) {432 if (arg_idx == argc-1 || !parse_tensor_type(argv[++arg_idx], tensor_type_opts)) {433 usage(argv[0]);434 }435 } else if (strcmp(argv[arg_idx], "--tensor-type-file") == 0) {436 if (arg_idx == argc-1 || !parse_tensor_type_file(argv[++arg_idx], tensor_type_opts)) {437 usage(argv[0]);438 }439 } else if (strcmp(argv[arg_idx], "--prune-layers") == 0) {440 if (arg_idx == argc-1 || !parse_layer_prune(argv[++arg_idx], prune_layers)) {441 usage(argv[0]);442 }443 } else if (strcmp(argv[arg_idx], "--override-kv") == 0) {444 if (arg_idx == argc-1 || !string_parse_kv_override(argv[++arg_idx], kv_overrides)) {445 usage(argv[0]);446 }447 } else if (strcmp(argv[arg_idx], "--dry-run") == 0) {448 params.dry_run = true;449 } else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) {450 params.allow_requantize = true;451 } else if (strcmp(argv[arg_idx], "--pure") == 0) {452 params.pure = true;453 } else if (strcmp(argv[arg_idx], "--imatrix") == 0) {454 if (arg_idx < argc-1) {455 imatrix_file = argv[++arg_idx];456 } else {457 usage(argv[0]);458 }459 } else if (strcmp(argv[arg_idx], "--include-weights") == 0) {460 if (arg_idx < argc-1) {461 included_weights.emplace_back(argv[++arg_idx]);462 } else {463 usage(argv[0]);464 }465 } else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) {466 if (arg_idx < argc-1) {467 excluded_weights.emplace_back(argv[++arg_idx]);468 } else {469 usage(argv[0]);470 }471 } else if (strcmp(argv[arg_idx], "--keep-split") == 0) {472 params.keep_split = true;473 } else if (strcmp(argv[arg_idx], "--max-buffer-size") == 0) {474 if (arg_idx == argc-1) {475 usage(argv[0]);476 }477 const int mib = atoi(argv[++arg_idx]);478 if (mib <= 0) {479 fprintf(stderr, "%s: invalid --max-buffer-size '%s'\n", __func__, argv[arg_idx]);480 return 1;481 }482 params.max_buf_size = (size_t) mib * 1024 * 1024;483 } else {484 usage(argv[0]);485 }486 }487 488 if (argc - arg_idx < 2) {489 printf("%s: bad arguments\n", argv[0]);490 usage(argv[0]);491 }492 if (!included_weights.empty() && !excluded_weights.empty()) {493 usage(argv[0]);494 }495 496 std::vector<std::string> imatrix_datasets;497 std::unordered_map<std::string, std::vector<float>> imatrix_data;498 int m_last_call = prepare_imatrix(imatrix_file, imatrix_datasets, included_weights, excluded_weights, imatrix_data);499 500 std::vector<llama_model_imatrix_data> i_data;501 std::vector<llama_model_tensor_override> t_override;502 if (!imatrix_data.empty()) {503 i_data.reserve(imatrix_data.size() + 1);504 for (const auto & kv : imatrix_data) {505 i_data.push_back({kv.first.c_str(), kv.second.data(), kv.second.size()});506 }507 i_data.push_back({nullptr, nullptr, 0}); // array terminator508 params.imatrix = i_data.data();509 {510 llama_model_kv_override kvo;511 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_FILE);512 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;513 strncpy(kvo.val_str, imatrix_file.c_str(), 127);514 kvo.val_str[127] = '\0';515 kv_overrides.emplace_back(std::move(kvo));516 }517 if (!imatrix_datasets.empty()) {518 llama_model_kv_override kvo;519 // TODO: list multiple datasets when there are more than one520 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_DATASET);521 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;522 strncpy(kvo.val_str, imatrix_datasets[0].c_str(), 127);523 kvo.val_str[127] = '\0';524 kv_overrides.emplace_back(std::move(kvo));525 }526 {527 llama_model_kv_override kvo;528 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES);529 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;530 kvo.val_i64 = imatrix_data.size();531 kv_overrides.emplace_back(std::move(kvo));532 }533 if (m_last_call > 0) {534 llama_model_kv_override kvo;535 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS);536 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;537 kvo.val_i64 = m_last_call;538 kv_overrides.emplace_back(std::move(kvo));539 }540 }541 if (!kv_overrides.empty()) {542 kv_overrides.emplace_back();543 kv_overrides.back().key[0] = 0;544 params.kv_overrides = kv_overrides.data();545 }546 if (!tensor_type_opts.empty()) {547 t_override.reserve(tensor_type_opts.size() + 1);548 for (const auto & tt : tensor_type_opts) {549 t_override.push_back({tt.name.c_str(), tt.type});550 }551 t_override.push_back({nullptr, GGML_TYPE_COUNT}); // array terminator552 params.tt_overrides = t_override.data();553 }554 if (!prune_layers.empty()) {555 prune_layers.push_back(-1); // array terminator556 params.prune_layers = prune_layers.data();557 }558 559 llama_backend_init();560 561 // parse command line arguments562 const std::string fname_inp = argv[arg_idx];563 arg_idx++;564 std::string fname_out;565 566 std::string ftype_str;567 std::string suffix = ".gguf";568 if (try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {569 // argv[arg_idx] is the ftype directly: <input> <ftype>570 if (!params.dry_run) {571 std::string fpath;572 const size_t pos = fname_inp.find_last_of("/\\");573 if (pos != std::string::npos) {574 fpath = fname_inp.substr(0, pos + 1);575 }576 577 // export as [inp path]/ggml-model-[ftype]. Only add extension if there is no splitting578 fname_out = fpath + "ggml-model-" + ftype_str;579 if (!params.keep_split) {580 fname_out += suffix;581 }582 }583 arg_idx++;584 if (ftype_str == "COPY") {585 params.only_copy = true;586 }587 } else {588 // argv[arg_idx] is not a valid ftype, so treat it as output path: <input> <output> <ftype>589 fname_out = argv[arg_idx];590 if (params.keep_split && fname_out.find(suffix) != std::string::npos) {591 fname_out = fname_out.substr(0, fname_out.length() - suffix.length());592 }593 arg_idx++;594 595 if (argc <= arg_idx) {596 fprintf(stderr, "%s: missing ftype\n", __func__);597 return 1;598 }599 if (!try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {600 fprintf(stderr, "%s: invalid ftype '%s'\n", __func__, argv[arg_idx]);601 return 1;602 }603 if (ftype_str == "COPY") {604 params.only_copy = true;605 }606 arg_idx++;607 }608 609 // parse nthreads610 if (argc > arg_idx) {611 try {612 params.nthread = std::stoi(argv[arg_idx]);613 }614 catch (const std::exception & e) {615 fprintf(stderr, "%s: invalid nthread '%s' (%s)\n", __func__, argv[arg_idx], e.what());616 return 1;617 }618 }619 620 if (!params.dry_run) {621 if (std::error_code ec; std::filesystem::equivalent(fname_inp, fname_out, ec)) {622 fprintf(stderr, "%s: error: input and output files are the same: '%s'\n", __func__, fname_inp.c_str());623 return 1;624 }625 }626 627 llama_print_build_info(llama_version());628 629 if (params.dry_run) {630 fprintf(stderr, "%s: calculating quantization size for '%s' as %s", __func__, fname_inp.c_str(), ftype_str.c_str());631 } else {632 fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str());633 }634 635 if (params.nthread > 0) {636 fprintf(stderr, " using %d threads", params.nthread);637 }638 fprintf(stderr, "\n");639 640 const int64_t t_main_start_us = llama_time_us();641 642 int64_t t_quantize_us = 0;643 644 // load the model645 {646 const int64_t t_start_us = llama_time_us();647 648 if (llama_model_quantize(fname_inp.c_str(), fname_out.c_str(), ¶ms)) {649 fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());650 return 1;651 }652 653 t_quantize_us = llama_time_us() - t_start_us;654 }655 656 // report timing657 {658 const int64_t t_main_end_us = llama_time_us();659 660 printf("\n");661 printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0);662 printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0);663 }664 665 llama_backend_free();666 667 return 0;668}669