echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0762
1#include "llama.h"2 3#include "build-info.h"4#include "common.h"5 6#include "gguf.h"7 8#include <algorithm>9#include <cctype>10#include <clocale>11#include <cmath>12#include <cstdio>13#include <cstring>14#include <vector>15#include <string>16#include <unordered_map>17#include <map>18#include <fstream>19#include <filesystem>20 21// result of parsing --tensor-type option22// changes to this struct must also be reflected in src/llama-quant.cpp23struct tensor_type_option {24 std::string name;25 ggml_type type = GGML_TYPE_COUNT;26};27 28struct quant_option {29 std::string name;30 llama_ftype ftype;31 std::string desc;32};33 34static const std::vector<quant_option> QUANT_OPTIONS = {35 { "Q1_0", LLAMA_FTYPE_MOSTLY_Q1_0, " 1.125 bpw quantization", },36 { "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0, " 4.34G, +0.4685 ppl @ Llama-3-8B", },37 { "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 4.78G, +0.4511 ppl @ Llama-3-8B", },38 { "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE", },39 { "Q5_0", LLAMA_FTYPE_MOSTLY_Q5_0, " 5.21G, +0.1316 ppl @ Llama-3-8B", },40 { "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 5.65G, +0.1062 ppl @ Llama-3-8B", },41 { "IQ2_XXS", LLAMA_FTYPE_MOSTLY_IQ2_XXS, " 2.06 bpw quantization", },42 { "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", },43 { "IQ2_S", LLAMA_FTYPE_MOSTLY_IQ2_S, " 2.5 bpw quantization", },44 { "IQ2_M", LLAMA_FTYPE_MOSTLY_IQ2_M, " 2.7 bpw quantization", },45 { "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S, " 1.56 bpw quantization", },46 { "IQ1_M", LLAMA_FTYPE_MOSTLY_IQ1_M, " 1.75 bpw quantization", },47 { "TQ1_0", LLAMA_FTYPE_MOSTLY_TQ1_0, " 1.69 bpw ternarization", },48 { "TQ2_0", LLAMA_FTYPE_MOSTLY_TQ2_0, " 2.06 bpw ternarization", },49 { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.96G, +3.5199 ppl @ Llama-3-8B", },50 { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.96G, +3.1836 ppl @ Llama-3-8B", },51 { "IQ3_XXS", LLAMA_FTYPE_MOSTLY_IQ3_XXS, " 3.06 bpw quantization", },52 { "IQ3_S", LLAMA_FTYPE_MOSTLY_IQ3_S, " 3.44 bpw quantization", },53 { "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M, " 3.66 bpw quantization mix", },54 { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" },55 { "IQ3_XS", LLAMA_FTYPE_MOSTLY_IQ3_XS, " 3.3 bpw quantization", },56 { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 3.41G, +1.6321 ppl @ Llama-3-8B", },57 { "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.74G, +0.6569 ppl @ Llama-3-8B", },58 { "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 4.03G, +0.5562 ppl @ Llama-3-8B", },59 { "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL, " 4.50 bpw non-linear quantization", },60 { "IQ4_XS", LLAMA_FTYPE_MOSTLY_IQ4_XS, " 4.25 bpw non-linear quantization", },61 { "Q4_K", LLAMA_FTYPE_MOSTLY_Q4_K_M, "alias for Q4_K_M", },62 { "Q4_K_S", LLAMA_FTYPE_MOSTLY_Q4_K_S, " 4.37G, +0.2689 ppl @ Llama-3-8B", },63 { "Q4_K_M", LLAMA_FTYPE_MOSTLY_Q4_K_M, " 4.58G, +0.1754 ppl @ Llama-3-8B", },64 { "Q5_K", LLAMA_FTYPE_MOSTLY_Q5_K_M, "alias for Q5_K_M", },65 { "Q5_K_S", LLAMA_FTYPE_MOSTLY_Q5_K_S, " 5.21G, +0.1049 ppl @ Llama-3-8B", },66 { "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 5.33G, +0.0569 ppl @ Llama-3-8B", },67 { "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 6.14G, +0.0217 ppl @ Llama-3-8B", },68 { "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0, " 7.96G, +0.0026 ppl @ Llama-3-8B", },69 { "F16", LLAMA_FTYPE_MOSTLY_F16, "14.00G, +0.0020 ppl @ Mistral-7B", },70 { "BF16", LLAMA_FTYPE_MOSTLY_BF16, "14.00G, -0.0050 ppl @ Mistral-7B", },71 { "F32", LLAMA_FTYPE_ALL_F32, "26.00G @ 7B", },72 // Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.73 { "COPY", LLAMA_FTYPE_ALL_F32, "only copy tensors, no quantizing", },74};75 76static const char * const LLM_KV_QUANTIZE_IMATRIX_FILE = "quantize.imatrix.file";77static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrix.dataset";78static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";79static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";80 81// TODO: share with imatrix.cpp82static const char * const LLM_KV_IMATRIX_DATASETS = "imatrix.datasets";83static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";84static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size";85 86static bool striequals(const char * a, const char * b) {87 while (*a && *b) {88 if (std::tolower(*a) != std::tolower(*b)) {89 return false;90 }91 a++; b++;92 }93 return *a == *b;94}95 96static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std::string & ftype_str_out) {97 std::string ftype_str;98 99 for (auto ch : ftype_str_in) {100 ftype_str.push_back(std::toupper(ch));101 }102 for (const auto & it : QUANT_OPTIONS) {103 if (striequals(it.name.c_str(), ftype_str.c_str())) {104 ftype = it.ftype;105 ftype_str_out = it.name;106 return true;107 }108 }109 try {110 int ftype_int = std::stoi(ftype_str);111 for (const auto & it : QUANT_OPTIONS) {112 if (it.ftype == ftype_int) {113 ftype = it.ftype;114 ftype_str_out = it.name;115 return true;116 }117 }118 }119 catch (...) {120 // stoi failed121 }122 return false;123}124 125[[noreturn]]126static void usage(const char * executable) {127 printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);128 printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");129 printf(" [--prune-layers] [--keep-split] [--override-kv] [--dry-run]\n");130 printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");131 printf(" --allow-requantize\n");132 printf(" allow requantizing tensors that have already been quantized\n");133 printf(" WARNING: this can severely reduce quality compared to quantizing\n");134 printf(" from 16bit or 32bit!\n");135 printf(" --leave-output-tensor\n");136 printf(" leave output.weight un(re)quantized\n");137 printf(" increases model size but may also increase quality, especially when requantizing\n");138 printf(" --pure\n");139 printf(" disable k-quant mixtures and quantize all tensors to the same type\n");140 printf(" --imatrix file_name\n");141 printf(" use data in file_name as importance matrix for quant optimizations\n");142 printf(" --include-weights tensor_name\n");143 printf(" use importance matrix for this/these tensor(s)\n");144 printf(" --exclude-weights tensor_name\n");145 printf(" do not use importance matrix for this/these tensor(s)\n");146 printf(" --output-tensor-type ggml_type\n");147 printf(" use this ggml_type for the output.weight tensor\n");148 printf(" --token-embedding-type ggml_type\n");149 printf(" use this ggml_type for the token embeddings tensor\n");150 printf(" --tensor-type tensor_name=ggml_type\n");151 printf(" quantize this tensor to this ggml_type\n");152 printf(" this is an advanced option to selectively quantize tensors. may be specified multiple times.\n");153 printf(" example: --tensor-type attn_q=q8_0\n");154 printf(" --tensor-type-file tensor_types.txt\n");155 printf(" list of tensors to quantize to a specific ggml_type\n");156 printf(" this is an advanced option to selectively quantize a long list of tensors.\n");157 printf(" the file should use the same format as above, separated by spaces or newlines.\n");158 printf(" --prune-layers L0,L1,L2...\n");159 printf(" comma-separated list of layer numbers to prune from the model\n");160 printf(" WARNING: this is an advanced option, use with care.\n");161 printf(" --keep-split\n");162 printf(" generate quantized model in the same shards as input\n");163 printf(" --override-kv KEY=TYPE:VALUE\n");164 printf(" override model metadata by key in the quantized model. may be specified multiple times.\n");165 printf(" WARNING: this is an advanced option, use with care.\n");166 printf(" --dry-run\n");167 printf(" calculate and show the final quantization size without performing quantization\n");168 printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n");169 printf("note: --include-weights and --exclude-weights cannot be used together\n\n");170 printf("-----------------------------------------------------------------------------\n");171 printf(" allowed quantization types\n");172 printf("-----------------------------------------------------------------------------\n\n");173 for (const auto & it : QUANT_OPTIONS) {174 if (it.name != "COPY") {175 printf(" %2d or ", it.ftype);176 } else {177 printf(" ");178 }179 printf("%-7s : %s\n", it.name.c_str(), it.desc.c_str());180 }181 exit(1);182}183 184static int load_legacy_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {185 std::ifstream in(imatrix_file.c_str(), std::ios::binary);186 if (!in) {187 printf("%s: failed to open %s\n",__func__, imatrix_file.c_str());188 exit(1);189 }190 int n_entries;191 in.read((char *)&n_entries, sizeof(n_entries));192 if (in.fail() || n_entries < 1) {193 printf("%s: no data in file %s\n", __func__, imatrix_file.c_str());194 exit(1);195 }196 for (int i = 0; i < n_entries; ++i) {197 int len; in.read((char *)&len, sizeof(len));198 std::vector<char> name_as_vec(len+1);199 in.read((char *)name_as_vec.data(), len);200 if (in.fail()) {201 printf("%s: failed reading name for entry %d from %s\n", __func__, i+1, imatrix_file.c_str());202 exit(1);203 }204 name_as_vec[len] = 0;205 std::string name{name_as_vec.data()};206 auto & e = imatrix_data[name];207 int ncall;208 in.read((char *)&ncall, sizeof(ncall));209 int nval;210 in.read((char *)&nval, sizeof(nval));211 if (in.fail() || nval < 1) {212 printf("%s: failed reading number of values for entry %d\n", __func__, i);213 imatrix_data = {};214 exit(1);215 }216 e.resize(nval);217 in.read((char *)e.data(), nval*sizeof(float));218 if (in.fail()) {219 printf("%s: failed reading data for entry %d\n", __func__, i);220 imatrix_data = {};221 exit(1);222 }223 if (ncall > 0) {224 for (auto & v : e) {225 v /= ncall;226 }227 }228 229 if (getenv("LLAMA_TRACE")) {230 printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n", __func__, int(e.size()), ncall, name.c_str());231 }232 }233 234 // latest legacy imatrix version contains the dataset filename at the end of the file235 int m_last_call = 0;236 if (in.peek() != EOF) {237 in.read((char *)&m_last_call, sizeof(m_last_call));238 int dataset_len;239 in.read((char *)&dataset_len, sizeof(dataset_len));240 std::vector<char> dataset_as_vec(dataset_len);241 in.read(dataset_as_vec.data(), dataset_len);242 imatrix_datasets.resize(1);243 imatrix_datasets[0].assign(dataset_as_vec.begin(), dataset_as_vec.end());244 printf("%s: imatrix dataset='%s'\n", __func__, imatrix_datasets[0].c_str());245 }246 printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_call);247 return m_last_call;248}249 250static int load_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {251 252 struct ggml_context * ctx = nullptr;253 struct gguf_init_params meta_gguf_params = {254 /* .no_alloc = */ false, // the data is needed255 /* .ctx = */ &ctx,256 };257 struct gguf_context * ctx_gguf = gguf_init_from_file(imatrix_file.c_str(), meta_gguf_params);258 if (!ctx_gguf) {259 fprintf(stderr, "%s: imatrix file '%s' is using old format\n", __func__, imatrix_file.c_str());260 return load_legacy_imatrix(imatrix_file, imatrix_datasets, imatrix_data);261 }262 const int32_t n_entries = gguf_get_n_tensors(ctx_gguf);263 if (n_entries < 1) {264 fprintf(stderr, "%s: no data in file %s\n", __func__, imatrix_file.c_str());265 gguf_free(ctx_gguf);266 ggml_free(ctx);267 exit(1);268 }269 270 const int dataset_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);271 const int chunk_count_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);272 const int chunk_size_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);273 if (dataset_idx < 0 || chunk_count_idx < 0 || chunk_size_idx < 0) {274 fprintf(stderr, "%s: missing imatrix metadata in file %s\n", __func__, imatrix_file.c_str());275 gguf_free(ctx_gguf);276 ggml_free(ctx);277 exit(1);278 }279 280 const uint32_t chunk_size = gguf_get_val_u32(ctx_gguf, chunk_size_idx);281 282 const std::string sums_suffix{ ".in_sum2" };283 const std::string counts_suffix{ ".counts" };284 285 // Using an ordered map to get a deterministic iteration order.286 std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;287 288 for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {289 std::string name = cur->name;290 291 if (name.empty()) { continue; }292 293 if (string_remove_suffix(name, sums_suffix)) {294 // in_sum2295 sums_counts_for[std::move(name)].first = cur;296 } else if (string_remove_suffix(name, counts_suffix)) {297 // counts298 sums_counts_for[std::move(name)].second = cur;299 } else {300 // ignore other tensors301 }302 }303 304 for (const auto & sc : sums_counts_for) {305 const std::string & name = sc.first;306 const struct ggml_tensor * sums = sc.second.first;307 const struct ggml_tensor * counts = sc.second.second;308 309 if (!sums || !counts) {310 fprintf(stderr, "%s: mismatched sums and counts for %s\n", __func__, name.c_str());311 gguf_free(ctx_gguf);312 ggml_free(ctx);313 exit(1);314 }315 316 const int64_t ne0 = sums->ne[0];317 const int64_t ne1 = sums->ne[1];318 319 auto & e = imatrix_data[name];320 e.resize(ggml_nelements(sums));321 float max_count = 0.0f;322 for (int64_t j = 0; j < ne1; ++j) {323 const float count = ((const float *) counts->data)[j];324 if (count > 0.0f) {325 for (int64_t i = 0; i < ne0; ++i) {326 e[j*ne0 + i] = ((const float *) sums->data)[j*ne0 + i] / count;327 }328 } else {329 // Partial imatrix data, this tensor never got any input during calibration330 for (int64_t i = 0; i < ne0; ++i) {331 e[j*ne0 + i] = 1;332 }333 }334 if (count > max_count) {335 max_count = count;336 }337 }338 if (getenv("LLAMA_TRACE")) {339 printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n", __func__, int(e.size()), int(max_count), int(max_count / chunk_size), name.c_str());340 }341 }342 343 int m_last_chunk = gguf_get_val_u32(ctx_gguf, chunk_count_idx);344 345 int64_t n_datasets = gguf_get_arr_n(ctx_gguf, dataset_idx);346 imatrix_datasets.reserve(n_datasets);347 for (int64_t i = 0; i < n_datasets; ++i) {348 imatrix_datasets.push_back(gguf_get_arr_str(ctx_gguf, dataset_idx, i));349 }350 printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());351 for (size_t i = 1; i < imatrix_datasets.size(); ++i) {352 printf(", '%s'", imatrix_datasets[i].c_str());353 }354 printf("]\n");355 356 printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_chunk);357 358 gguf_free(ctx_gguf);359 ggml_free(ctx);360 361 return m_last_chunk;362}363 364static int prepare_imatrix(const std::string & imatrix_file,365 std::vector<std::string> & imatrix_dataset,366 const std::vector<std::string> & included_weights,367 const std::vector<std::string> & excluded_weights,368 std::unordered_map<std::string, std::vector<float>> & imatrix_data) {369 int m_last_call = -1;370 if (!imatrix_file.empty()) {371 m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data);372 }373 if (imatrix_data.empty()) {374 return m_last_call;375 }376 if (!excluded_weights.empty()) {377 for (const auto & name : excluded_weights) {378 for (auto it = imatrix_data.begin(); it != imatrix_data.end();) {379 auto pos = it->first.find(name);380 if (pos != std::string::npos) {381 it = imatrix_data.erase(it);382 } else {383 ++it;384 }385 }386 }387 }388 if (!included_weights.empty()) {389 std::unordered_map<std::string, std::vector<float>> tmp;390 for (const auto & name : included_weights) {391 for (auto & e : imatrix_data) {392 auto pos = e.first.find(name);393 if (pos != std::string::npos) {394 tmp.emplace(std::move(e));395 }396 }397 }398 imatrix_data = std::move(tmp);399 }400 if (!imatrix_data.empty()) {401 printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size()));402 }403 return m_last_call;404}405 406static ggml_type parse_ggml_type(const char * arg) {407 for (int i = 0; i < GGML_TYPE_COUNT; ++i) {408 auto type = (ggml_type)i;409 const auto * name = ggml_type_name(type);410 if (name && striequals(name, arg)) {411 return type;412 }413 }414 fprintf(stderr, "\n%s: invalid ggml_type '%s'\n\n", __func__, arg);415 return GGML_TYPE_COUNT;416}417 418static bool parse_tensor_type(const char * data, std::vector<tensor_type_option> & tensor_type) {419 const char * sep = strchr(data, '=');420 if (sep == nullptr) {421 printf("\n%s: malformed tensor type '%s'\n\n", __func__, data);422 return false;423 }424 425 const size_t tn_len = sep - data;426 if (tn_len == 0) {427 printf("\n%s: missing tensor name\n\n", __func__);428 return false;429 }430 if (const size_t qt_len = strlen(sep); qt_len == 1) {431 printf("\n%s: missing quantization type\n\n", __func__);432 return false;433 }434 435 std::string tn(data, tn_len);436 std::transform(tn.begin(), tn.end(), tn.begin(), tolower);437 sep++;438 tensor_type_option tensor_type_opt;439 tensor_type_opt.name = tn;440 tensor_type_opt.type = parse_ggml_type(sep);441 tensor_type.emplace_back(std::move(tensor_type_opt));442 if (tensor_type_opt.type == GGML_TYPE_COUNT) {443 printf("\n%s: invalid quantization type '%s'\n\n", __func__, sep);444 return false;445 }446 447 return true;448}449 450static bool parse_tensor_type_file(const char * filename, std::vector<tensor_type_option> & tensor_type) {451 std::ifstream file(filename);452 if (!file) {453 printf("\n%s: failed to open file '%s': %s\n\n", __func__, filename, std::strerror(errno));454 return false;455 }456 457 std::string arg;458 while (file >> arg) {459 if (!parse_tensor_type(arg.c_str(), tensor_type)) {460 return false;461 }462 }463 464 return true;465}466 467static bool parse_layer_prune(const char * data, std::vector<int> & prune_layers) {468 if (!data) {469 printf("\n%s: no layer pruning ids provided\n\n", __func__);470 return false;471 }472 473 const auto block_ids = string_split<std::string>(data, ',');474 for (const auto & block_id : block_ids) {475 int id;476 try {477 id = std::stoi(block_id);478 } catch (...) {479 id = -1;480 }481 if (id < 0) {482 printf("\n%s: invalid layer id '%s'\n\n", __func__, block_id.c_str());483 return false;484 }485 prune_layers.emplace_back(id);486 }487 488 sort(prune_layers.begin(), prune_layers.end());489 prune_layers.erase(std::unique(prune_layers.begin(), prune_layers.end()), prune_layers.end());490 return true;491}492 493int main(int argc, char ** argv) {494 std::setlocale(LC_NUMERIC, "C");495 if (argc < 3) {496 usage(argv[0]);497 }498 499 llama_model_quantize_params params = llama_model_quantize_default_params();500 501 int arg_idx = 1;502 std::string imatrix_file;503 std::vector<std::string> included_weights, excluded_weights;504 std::vector<llama_model_kv_override> kv_overrides;505 std::vector<tensor_type_option> tensor_type_opts;506 std::vector<int> prune_layers;507 508 for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {509 if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) {510 params.quantize_output_tensor = false;511 } else if (strcmp(argv[arg_idx], "--output-tensor-type") == 0) {512 if (arg_idx < argc-1) {513 params.output_tensor_type = parse_ggml_type(argv[++arg_idx]);514 if (params.output_tensor_type == GGML_TYPE_COUNT) {515 usage(argv[0]);516 }517 } else {518 usage(argv[0]);519 }520 } else if (strcmp(argv[arg_idx], "--token-embedding-type") == 0) {521 if (arg_idx < argc-1) {522 params.token_embedding_type = parse_ggml_type(argv[++arg_idx]);523 if (params.token_embedding_type == GGML_TYPE_COUNT) {524 usage(argv[0]);525 }526 } else {527 usage(argv[0]);528 }529 } else if (strcmp(argv[arg_idx], "--tensor-type") == 0) {530 if (arg_idx == argc-1 || !parse_tensor_type(argv[++arg_idx], tensor_type_opts)) {531 usage(argv[0]);532 }533 } else if (strcmp(argv[arg_idx], "--tensor-type-file") == 0) {534 if (arg_idx == argc-1 || !parse_tensor_type_file(argv[++arg_idx], tensor_type_opts)) {535 usage(argv[0]);536 }537 } else if (strcmp(argv[arg_idx], "--prune-layers") == 0) {538 if (arg_idx == argc-1 || !parse_layer_prune(argv[++arg_idx], prune_layers)) {539 usage(argv[0]);540 }541 } else if (strcmp(argv[arg_idx], "--override-kv") == 0) {542 if (arg_idx == argc-1 || !string_parse_kv_override(argv[++arg_idx], kv_overrides)) {543 usage(argv[0]);544 }545 } else if (strcmp(argv[arg_idx], "--dry-run") == 0) {546 params.dry_run = true;547 } else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) {548 params.allow_requantize = true;549 } else if (strcmp(argv[arg_idx], "--pure") == 0) {550 params.pure = true;551 } else if (strcmp(argv[arg_idx], "--imatrix") == 0) {552 if (arg_idx < argc-1) {553 imatrix_file = argv[++arg_idx];554 } else {555 usage(argv[0]);556 }557 } else if (strcmp(argv[arg_idx], "--include-weights") == 0) {558 if (arg_idx < argc-1) {559 included_weights.emplace_back(argv[++arg_idx]);560 } else {561 usage(argv[0]);562 }563 } else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) {564 if (arg_idx < argc-1) {565 excluded_weights.emplace_back(argv[++arg_idx]);566 } else {567 usage(argv[0]);568 }569 } else if (strcmp(argv[arg_idx], "--keep-split") == 0) {570 params.keep_split = true;571 } else {572 usage(argv[0]);573 }574 }575 576 if (argc - arg_idx < 2) {577 printf("%s: bad arguments\n", argv[0]);578 usage(argv[0]);579 }580 if (!included_weights.empty() && !excluded_weights.empty()) {581 usage(argv[0]);582 }583 584 std::vector<std::string> imatrix_datasets;585 std::unordered_map<std::string, std::vector<float>> imatrix_data;586 int m_last_call = prepare_imatrix(imatrix_file, imatrix_datasets, included_weights, excluded_weights, imatrix_data);587 588 std::vector<llama_model_imatrix_data> i_data;589 std::vector<llama_model_tensor_override> t_override;590 if (!imatrix_data.empty()) {591 i_data.reserve(imatrix_data.size() + 1);592 for (const auto & kv : imatrix_data) {593 i_data.push_back({kv.first.c_str(), kv.second.data(), kv.second.size()});594 }595 i_data.push_back({nullptr, nullptr, 0}); // array terminator596 params.imatrix = i_data.data();597 {598 llama_model_kv_override kvo;599 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_FILE);600 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;601 strncpy(kvo.val_str, imatrix_file.c_str(), 127);602 kvo.val_str[127] = '\0';603 kv_overrides.emplace_back(std::move(kvo));604 }605 if (!imatrix_datasets.empty()) {606 llama_model_kv_override kvo;607 // TODO: list multiple datasets when there are more than one608 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_DATASET);609 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;610 strncpy(kvo.val_str, imatrix_datasets[0].c_str(), 127);611 kvo.val_str[127] = '\0';612 kv_overrides.emplace_back(std::move(kvo));613 }614 {615 llama_model_kv_override kvo;616 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES);617 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;618 kvo.val_i64 = imatrix_data.size();619 kv_overrides.emplace_back(std::move(kvo));620 }621 if (m_last_call > 0) {622 llama_model_kv_override kvo;623 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS);624 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;625 kvo.val_i64 = m_last_call;626 kv_overrides.emplace_back(std::move(kvo));627 }628 }629 if (!kv_overrides.empty()) {630 kv_overrides.emplace_back();631 kv_overrides.back().key[0] = 0;632 params.kv_overrides = kv_overrides.data();633 }634 if (!tensor_type_opts.empty()) {635 t_override.reserve(tensor_type_opts.size() + 1);636 for (const auto & tt : tensor_type_opts) {637 t_override.push_back({tt.name.c_str(), tt.type});638 }639 t_override.push_back({nullptr, GGML_TYPE_COUNT}); // array terminator640 params.tt_overrides = t_override.data();641 }642 if (!prune_layers.empty()) {643 prune_layers.push_back(-1); // array terminator644 params.prune_layers = prune_layers.data();645 }646 647 llama_backend_init();648 649 // parse command line arguments650 const std::string fname_inp = argv[arg_idx];651 arg_idx++;652 std::string fname_out;653 654 std::string ftype_str;655 std::string suffix = ".gguf";656 if (try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {657 // argv[arg_idx] is the ftype directly: <input> <ftype>658 if (!params.dry_run) {659 std::string fpath;660 const size_t pos = fname_inp.find_last_of("/\\");661 if (pos != std::string::npos) {662 fpath = fname_inp.substr(0, pos + 1);663 }664 665 // export as [inp path]/ggml-model-[ftype]. Only add extension if there is no splitting666 fname_out = fpath + "ggml-model-" + ftype_str;667 if (!params.keep_split) {668 fname_out += suffix;669 }670 }671 arg_idx++;672 if (ftype_str == "COPY") {673 params.only_copy = true;674 }675 } else {676 // argv[arg_idx] is not a valid ftype, so treat it as output path: <input> <output> <ftype>677 fname_out = argv[arg_idx];678 if (params.keep_split && fname_out.find(suffix) != std::string::npos) {679 fname_out = fname_out.substr(0, fname_out.length() - suffix.length());680 }681 arg_idx++;682 683 if (argc <= arg_idx) {684 fprintf(stderr, "%s: missing ftype\n", __func__);685 return 1;686 }687 if (!try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {688 fprintf(stderr, "%s: invalid ftype '%s'\n", __func__, argv[arg_idx]);689 return 1;690 }691 if (ftype_str == "COPY") {692 params.only_copy = true;693 }694 arg_idx++;695 }696 697 // parse nthreads698 if (argc > arg_idx) {699 try {700 params.nthread = std::stoi(argv[arg_idx]);701 }702 catch (const std::exception & e) {703 fprintf(stderr, "%s: invalid nthread '%s' (%s)\n", __func__, argv[arg_idx], e.what());704 return 1;705 }706 }707 708 if (!params.dry_run) {709 if (std::error_code ec; std::filesystem::equivalent(fname_inp, fname_out, ec)) {710 fprintf(stderr, "%s: error: input and output files are the same: '%s'\n", __func__, fname_inp.c_str());711 return 1;712 }713 }714 715 llama_print_build_info();716 717 if (params.dry_run) {718 fprintf(stderr, "%s: calculating quantization size for '%s' as %s", __func__, fname_inp.c_str(), ftype_str.c_str());719 } else {720 fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str());721 }722 723 if (params.nthread > 0) {724 fprintf(stderr, " using %d threads", params.nthread);725 }726 fprintf(stderr, "\n");727 728 const int64_t t_main_start_us = llama_time_us();729 730 int64_t t_quantize_us = 0;731 732 // load the model733 {734 const int64_t t_start_us = llama_time_us();735 736 if (llama_model_quantize(fname_inp.c_str(), fname_out.c_str(), ¶ms)) {737 fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());738 return 1;739 }740 741 t_quantize_us = llama_time_us() - t_start_us;742 }743 744 // report timing745 {746 const int64_t t_main_end_us = llama_time_us();747 748 printf("\n");749 printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0);750 printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0);751 }752 753 llama_backend_free();754 755 return 0;756}757