Felipe97/llama-cpp-compiled
01.1k
1#include "llama-impl.h"2#include "llama-model.h"3#include "llama-model-loader.h"4#include "llama-ext.h"5#include "llama.h"6 7#include <algorithm>8#include <cmath>9#include <cstring>10#include <cinttypes>11#include <fstream>12#include <mutex>13#include <regex>14#include <thread>15#include <unordered_map>16 17// result of parsing --tensor-type option18// (changes to this struct must be reflected in tools/quantize/quantize.cpp)19struct tensor_type_option {20 std::string name;21 ggml_type type = GGML_TYPE_COUNT;22};23 24// tensor categorization - used to avoid repeated string matching in quantization logic.25// this is different from LLM_TN - we want broad categories, not specific tensor names per arch.26enum class tensor_category {27 TOKEN_EMBD,28 ATTENTION_Q,29 ATTENTION_V,30 ATTENTION_K,31 ATTENTION_QKV,32 ATTENTION_KV_B,33 ATTENTION_OUTPUT,34 FFN_UP,35 FFN_GATE,36 FFN_DOWN,37 OUTPUT,38 OTHER39};40 41// max amount of tensor data kept in memory while quantizing a single tensor42static const size_t LLAMA_QUANT_MAX_BUF_SIZE = 8ull*1024*1024*1024;43 44static void zeros(std::ofstream & file, size_t n) {45 char zero = 0;46 for (size_t i = 0; i < n; ++i) {47 file.write(&zero, 1);48 }49}50 51static std::string remap_layer(const std::string & orig_name, const std::vector<int> & prune, std::map<int, std::string> & mapped, int & next_id) {52 if (prune.empty()) {53 return orig_name;54 }55 56 static const std::regex pattern(R"(blk\.(\d+)\.)");57 if (std::smatch match; std::regex_search(orig_name, match, pattern)) {58 const int blk = std::stoi(match[1]);59 std::string new_name = orig_name;60 61 if (mapped.count(blk)) {62 // Already mapped, do nothing63 } else if (std::find(prune.begin(), prune.end(), blk) != prune.end()) {64 mapped[blk] = "";65 } else if (blk < prune.front()) {66 mapped[blk] = std::to_string(blk);67 next_id = blk + 1;68 } else {69 mapped[blk] = std::to_string(next_id);70 ++next_id;71 }72 73 return mapped[blk].empty() ? mapped[blk] : new_name.replace(match.position(1), match.length(1), mapped[blk]);74 }75 76 return orig_name;77}78 79static std::string remap_imatrix(const std::string & orig_name, const std::map<int, std::string> & mapped) {80 if (mapped.empty()) {81 return orig_name;82 }83 84 static const std::regex pattern(R"(blk\.(\d+)\.)");85 if (std::smatch match; std::regex_search(orig_name, match, pattern)) {86 const std::string blk(match[1]);87 std::string new_name = orig_name;88 89 for (const auto & p : mapped) {90 if (p.second == blk) {91 return new_name.replace(match.position(1), match.length(1), std::to_string(p.first));92 }93 }94 GGML_ABORT("\n%s: imatrix mapping error for %s\n", __func__, orig_name.c_str());95 }96 97 return orig_name;98}99 100//101// helper functions for tensor name matching102//103 104static bool tensor_name_match_token_embd(const char * tensor_name) {105 return std::strcmp(tensor_name, "token_embd.weight") == 0 ||106 std::strcmp(tensor_name, "per_layer_token_embd.weight") == 0;107}108 109static bool tensor_name_match_output_weight(const char * tensor_name) {110 return std::strcmp(tensor_name, "output.weight") == 0;111}112 113//114// tensor categorization for quantization115//116// (this is different from LLM_TN - we want broad categories, not specific tensor names per arch)117//118 119static tensor_category tensor_get_category(const std::string & tensor_name) {120 if (tensor_name_match_output_weight(tensor_name.c_str())) {121 return tensor_category::OUTPUT;122 }123 if (tensor_name_match_token_embd(tensor_name.c_str())) {124 return tensor_category::TOKEN_EMBD;125 }126 if (tensor_name.find("attn_qkv.weight") != std::string::npos) {127 return tensor_category::ATTENTION_QKV;128 }129 if (tensor_name.find("attn_kv_b.weight") != std::string::npos) {130 return tensor_category::ATTENTION_KV_B;131 }132 if (tensor_name.find("attn_v.weight") != std::string::npos) {133 return tensor_category::ATTENTION_V;134 }135 if (tensor_name.find("attn_k.weight") != std::string::npos) {136 return tensor_category::ATTENTION_K;137 }138 if (tensor_name.find("attn_q.weight") != std::string::npos) {139 return tensor_category::ATTENTION_Q;140 }141 if (tensor_name.find("attn_output.weight") != std::string::npos) {142 return tensor_category::ATTENTION_OUTPUT;143 }144 if (tensor_name.find("ffn_up") != std::string::npos) {145 return tensor_category::FFN_UP;146 }147 if (tensor_name.find("ffn_gate") != std::string::npos) {148 return tensor_category::FFN_GATE;149 }150 if (tensor_name.find("ffn_down") != std::string::npos) {151 return tensor_category::FFN_DOWN;152 }153 return tensor_category::OTHER;154}155 156// check if category is for attention-v-like tensors (more sensitive to quantization)157static bool category_is_attn_v(tensor_category cat) {158 return cat == tensor_category::ATTENTION_V ||159 cat == tensor_category::ATTENTION_QKV ||160 cat == tensor_category::ATTENTION_KV_B;161}162 163//164// quantization state165//166 167struct quantize_state_impl {168 const llama_model & model;169 const llama_model_quantize_params * params;170 171 int n_attention_wv = 0;172 int n_ffn_down = 0;173 int n_ffn_gate = 0;174 int n_ffn_up = 0;175 int i_attention_wv = 0;176 int i_ffn_down = 0;177 int i_ffn_gate = 0;178 int i_ffn_up = 0;179 180 int n_fallback = 0;181 182 bool has_imatrix = false;183 184 // used to figure out if a model has tied embeddings (tok_embd shares weights with output)185 bool has_tied_embeddings = true; // assume tied until we see output.weight186 187 // tensor type override patterns (compiled once, used twice)188 std::vector<std::pair<std::regex, ggml_type>> tensor_type_patterns;189 190 quantize_state_impl(const llama_model & model, const llama_model_quantize_params * params):191 model(model), params(params)192 {193 // compile regex patterns once - they are expensive194 if (params->tt_overrides) {195 for (const auto * p = params->tt_overrides; p->pattern != nullptr; p++) {196 tensor_type_patterns.emplace_back(std::regex(p->pattern), p->type);197 }198 }199 }200};201 202// per-tensor metadata, computed in the preliminary loop and used in the main loop203struct tensor_metadata {204 std::string name;205 ggml_type target_type;206 tensor_category category;207 std::string remapped_imatrix_name;208 bool allows_quantization;209 bool requires_imatrix;210};211 212//213// dequantization214//215 216static void llama_tensor_dequantize_impl(217 ggml_type type, const void * data, float * f32_output, std::vector<std::thread> & workers,218 const size_t nelements, const int nthread219) {220 const ggml_type_traits * qtype = ggml_get_type_traits(type);221 if (ggml_is_quantized(type)) {222 if (qtype->to_float == NULL) {223 throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(type)));224 }225 } else if (type != GGML_TYPE_F16 &&226 type != GGML_TYPE_BF16) {227 throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(type)));228 }229 230 if (nthread < 2) {231 if (type == GGML_TYPE_F16) {232 ggml_fp16_to_fp32_row((const ggml_fp16_t *)data, f32_output, nelements);233 } else if (type == GGML_TYPE_BF16) {234 ggml_bf16_to_fp32_row((const ggml_bf16_t *)data, f32_output, nelements);235 } else if (ggml_is_quantized(type)) {236 qtype->to_float(data, f32_output, nelements);237 } else {238 GGML_ABORT("fatal error"); // unreachable239 }240 return;241 }242 243 size_t block_size;244 if (type == GGML_TYPE_F16 ||245 type == GGML_TYPE_BF16) {246 block_size = 1;247 } else {248 block_size = (size_t)ggml_blck_size(type);249 }250 251 size_t block_size_bytes = ggml_type_size(type);252 253 GGML_ASSERT(nelements % block_size == 0);254 size_t nblocks = nelements / block_size;255 size_t blocks_per_thread = nblocks / nthread;256 size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count257 258 size_t in_buff_offs = 0;259 size_t out_buff_offs = 0;260 261 for (int tnum = 0; tnum < nthread; tnum++) {262 size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread263 size_t thr_elems = thr_blocks * block_size; // number of elements for this thread264 size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread265 266 auto compute = [qtype] (ggml_type typ, const uint8_t * inbuf, float * outbuf, int nels) {267 if (typ == GGML_TYPE_F16) {268 ggml_fp16_to_fp32_row((const ggml_fp16_t *)inbuf, outbuf, nels);269 } else if (typ == GGML_TYPE_BF16) {270 ggml_bf16_to_fp32_row((const ggml_bf16_t *)inbuf, outbuf, nels);271 } else {272 qtype->to_float(inbuf, outbuf, nels);273 }274 };275 workers.emplace_back(compute, type, (const uint8_t *) data + in_buff_offs, f32_output + out_buff_offs, thr_elems);276 in_buff_offs += thr_block_bytes;277 out_buff_offs += thr_elems;278 }279 for (auto & w : workers) { w.join(); }280 workers.clear();281}282 283//284// do we allow this tensor to be quantized?285//286 287static bool tensor_allows_quantization(const llama_model_quantize_params * params, llm_arch arch, const ggml_tensor * tensor) {288 // trivial checks first -- no string ops needed289 if (params->only_copy) return false;290 291 // quantize only 2D and 3D tensors (experts)292 if (ggml_n_dims(tensor) < 2) return false;293 294 const std::string name = ggml_get_name(tensor);295 296 // This used to be a regex, but <regex> has an extreme cost to compile times.297 bool quantize = name.rfind("weight") == name.size() - 6; // ends with 'weight'?298 299 // do not quantize norm tensors300 quantize &= name.find("_norm.weight") == std::string::npos;301 302 quantize &= params->quantize_output_tensor || name != "output.weight";303 304 // do not quantize expert gating tensors305 // NOTE: can't use LLM_TN here because the layer number is not known306 quantize &= name.find("ffn_gate_inp.weight") == std::string::npos;307 308 // do not quantize the i32 token-id -> expert-id routing table (DeepSeek-V4)309 quantize &= name.find("ffn_gate_tid2eid.weight") == std::string::npos;310 311 // these are very small (e.g. 4x4)312 quantize &= name.find("altup") == std::string::npos;313 quantize &= name.find("laurel") == std::string::npos;314 315 // these are not too big so keep them as it is316 quantize &= name.find("per_layer_model_proj") == std::string::npos;317 318 // do not quantize positional embeddings and token types (BERT)319 quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD, "weight");320 quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");321 322 // do not quantize Mamba/Kimi's small conv1d weights323 // NOTE: can't use LLM_TN here because the layer number is not known324 quantize &= name.find("ssm_conv1d") == std::string::npos;325 quantize &= name.find("shortconv.conv.weight") == std::string::npos;326 327 // do not quantize MiniMax's indexer projection weights, they are tiny328 quantize &= name.find("indexer.k_proj.weight") == std::string::npos;329 quantize &= name.find("indexer.q_proj.weight") == std::string::npos;330 331 // do not quantize RWKV's small yet 2D weights332 quantize &= name.find("time_mix_first.weight") == std::string::npos;333 quantize &= name.find("time_mix_w0.weight") == std::string::npos;334 quantize &= name.find("time_mix_w1.weight") == std::string::npos;335 quantize &= name.find("time_mix_w2.weight") == std::string::npos;336 quantize &= name.find("time_mix_v0.weight") == std::string::npos;337 quantize &= name.find("time_mix_v1.weight") == std::string::npos;338 quantize &= name.find("time_mix_v2.weight") == std::string::npos;339 quantize &= name.find("time_mix_a0.weight") == std::string::npos;340 quantize &= name.find("time_mix_a1.weight") == std::string::npos;341 quantize &= name.find("time_mix_a2.weight") == std::string::npos;342 quantize &= name.find("time_mix_g1.weight") == std::string::npos;343 quantize &= name.find("time_mix_g2.weight") == std::string::npos;344 quantize &= name.find("time_mix_decay_w1.weight") == std::string::npos;345 quantize &= name.find("time_mix_decay_w2.weight") == std::string::npos;346 quantize &= name.find("time_mix_lerp_fused.weight") == std::string::npos;347 348 // do not quantize relative position bias (T5)349 quantize &= name.find("attn_rel_b.weight") == std::string::npos;350 351 // do not quantize specific multimodal tensors352 quantize &= name.find(".position_embd") == std::string::npos;353 quantize &= name.find("sam.pos_embd") == std::string::npos;354 quantize &= name.find("sam.neck.") == std::string::npos;355 quantize &= name.find("sam.net_") == std::string::npos;356 quantize &= name.find(".rel_pos") == std::string::npos;357 quantize &= name.find(".patch_embd") == std::string::npos;358 quantize &= name.find(".patch_merger") == std::string::npos;359 360 // audio codebook361 quantize &= name.find("a.rvq.codebook") == std::string::npos;362 quantize &= name.find("mm.a.code_embd") == std::string::npos;363 364 return quantize;365}366 367//368// tensor type selection369//370 371// incompatible tensor shapes are handled here - fallback to a compatible type372static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tensor * t, const ggml_type target_type) {373 ggml_type return_type = target_type;374 375 const int64_t ncols = t->ne[0];376 const int64_t qk_k = ggml_blck_size(target_type);377 378 if (ncols % qk_k != 0) { // this tensor's shape is incompatible with this quant379 LLAMA_LOG_WARN("warning: %-36s - ncols %6" PRId64 " not divisible by %3" PRId64 " (required for type %7s) ",380 t->name, ncols, qk_k, ggml_type_name(target_type));381 ++qs.n_fallback;382 383 switch (target_type) {384 // types on the left: block size 256385 case GGML_TYPE_IQ1_S:386 case GGML_TYPE_IQ1_M:387 case GGML_TYPE_IQ2_XXS:388 case GGML_TYPE_IQ2_XS:389 case GGML_TYPE_IQ2_S:390 case GGML_TYPE_IQ3_XXS:391 case GGML_TYPE_IQ3_S: // types on the right: block size 32392 case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break;393 case GGML_TYPE_Q2_0:394 case GGML_TYPE_Q2_K:395 case GGML_TYPE_Q3_K:396 case GGML_TYPE_TQ1_0:397 case GGML_TYPE_TQ2_0: return_type = GGML_TYPE_Q4_0; break;398 case GGML_TYPE_Q4_K: return_type = GGML_TYPE_Q5_0; break;399 case GGML_TYPE_Q5_K: return_type = GGML_TYPE_Q5_1; break;400 case GGML_TYPE_Q6_K: return_type = GGML_TYPE_Q8_0; break;401 default:402 if (qk_k <= 32) {403 // the target is already a 32-block type, so there is no smaller block to demote to404 // the check below turns it into F16, as a 256-block type does when its fallback does not fit405 return_type = target_type;406 break;407 }408 throw std::runtime_error(format("no tensor type fallback is defined for type %s",409 ggml_type_name(target_type)));410 }411 if (ncols % ggml_blck_size(return_type) != 0) {412 //413 // the fallback return type is still not compatible for this tensor!414 //415 // most likely, this tensor's first dimension is not divisible by 32.416 // this is very rare. we can either abort the quantization, or417 // fallback to F16 / F32.418 //419 LLAMA_LOG_WARN("(WARNING: must use F16 due to unusual shape) ");420 return_type = GGML_TYPE_F16;421 }422 LLAMA_LOG_WARN("-> falling back to %7s\n", ggml_type_name(return_type));423 }424 return return_type;425}426 427// internal standard logic for selecting the target tensor type based on tensor category, ftype, and model arch428static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype, tensor_category category) {429 const std::string name = ggml_get_name(tensor);430 431 // TODO: avoid hardcoded tensor names - use the TN_* constants432 const llm_arch arch = qs.model.arch;433 434 auto use_more_bits = [](int i_layer, int n_layers) -> bool {435 return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;436 };437 const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);438 auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {439 if (n_expert > 1) {440 // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but occasionally randomly441 // sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work442 // for getting the current layer as I initially thought, and we need to resort to parsing the443 // tensor name.444 if (sscanf(name, "blk.%d.", &i_layer) != 1) {445 throw std::runtime_error(format("Failed to determine layer for tensor %s", name));446 }447 if (i_layer < 0 || i_layer >= n_layer) {448 throw std::runtime_error(format("Bad layer %d for tensor %s. Must be in [0, %d)", i_layer, name, n_layer));449 }450 }451 return std::make_pair(i_layer, n_layer);452 };453 454 // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings455 // with the quantization of the output tensor456 if (category == tensor_category::OUTPUT || (qs.has_tied_embeddings && category == tensor_category::TOKEN_EMBD)) {457 if (qs.params->output_tensor_type < GGML_TYPE_COUNT) {458 new_type = qs.params->output_tensor_type;459 } else {460 const int64_t nx = tensor->ne[0];461 const int64_t qk_k = ggml_blck_size(new_type);462 463 if (ftype == LLAMA_FTYPE_MOSTLY_MXFP4_MOE) {464 new_type = GGML_TYPE_Q8_0;465 }466 else if (arch == LLM_ARCH_FALCON || nx % qk_k != 0) {467 new_type = GGML_TYPE_Q8_0;468 }469 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||470 ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ||471 ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {472 new_type = GGML_TYPE_Q5_K;473 }474 else if (new_type != GGML_TYPE_Q8_0) {475 new_type = GGML_TYPE_Q6_K;476 }477 }478 } else if (ftype == LLAMA_FTYPE_MOSTLY_MXFP4_MOE) {479 // MoE tensors -> MXFP4480 // other tensors -> Q8_0481 // MLA projection tensors are also 3D, so match expert tensor roles explicitly.482 const bool is_bailingmoe3_expert = arch == LLM_ARCH_BAILINGMOE3 &&483 (category == tensor_category::FFN_UP ||484 category == tensor_category::FFN_GATE ||485 category == tensor_category::FFN_DOWN);486 if (tensor->ne[2] > 1 && (arch != LLM_ARCH_BAILINGMOE3 || is_bailingmoe3_expert)) {487 new_type = GGML_TYPE_MXFP4;488 } else {489 new_type = GGML_TYPE_Q8_0;490 }491 } else if (category == tensor_category::TOKEN_EMBD) {492 if (qs.params->token_embedding_type < GGML_TYPE_COUNT) {493 new_type = qs.params->token_embedding_type;494 } else {495 if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||496 ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {497 new_type = GGML_TYPE_Q2_K;498 }499 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) {500 new_type = GGML_TYPE_IQ3_S;501 }502 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {503 new_type = GGML_TYPE_IQ3_S;504 }505 else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) {506 new_type = GGML_TYPE_Q4_K;507 }508 }509 } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||510 ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {511 if (category_is_attn_v(category)) {512 if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;513 else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;514 ++qs.i_attention_wv;515 }516 else if (qs.model.hparams.n_expert == 8 && category == tensor_category::ATTENTION_K) {517 new_type = GGML_TYPE_Q4_K;518 }519 else if (category == tensor_category::FFN_DOWN) {520 if (qs.i_ffn_down < qs.n_ffn_down/8) {521 new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;522 }523 ++qs.i_ffn_down;524 }525 else if (category == tensor_category::ATTENTION_OUTPUT) {526 if (qs.model.hparams.n_expert == 8) {527 new_type = GGML_TYPE_Q5_K;528 } else {529 if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) new_type = GGML_TYPE_IQ2_XXS;530 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) new_type = GGML_TYPE_IQ3_S;531 }532 }533 } else if (category_is_attn_v(category)) {534 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {535 new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;536 }537 else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {538 new_type = GGML_TYPE_Q4_K;539 }540 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {541 new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_IQ3_S : GGML_TYPE_IQ3_XXS;542 }543 else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S) && qs.model.hparams.n_gqa() >= 4) {544 new_type = GGML_TYPE_Q4_K;545 }546 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {547 new_type = GGML_TYPE_Q4_K;548 }549 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {550 new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;551 }552 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;553 else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && qs.model.hparams.n_gqa() >= 4) {554 new_type = GGML_TYPE_Q5_K;555 }556 else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) &&557 use_more_bits(qs.i_attention_wv, qs.n_attention_wv)) new_type = GGML_TYPE_Q6_K;558 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) new_type = GGML_TYPE_Q5_K;559 if (qs.model.type == LLM_TYPE_70B) {560 // In the 70B model we have 8 heads sharing the same attn_v weights. As a result, the attn_v.weight tensor is561 // 8x smaller compared to attn_q.weight. Hence, we can get a nice boost in quantization accuracy with562 // nearly negligible increase in model size by quantizing this tensor with more bits:563 if (new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K) new_type = GGML_TYPE_Q5_K;564 }565 if (qs.model.hparams.n_expert == 8) {566 // for the 8-expert model, bumping this to Q8_0 trades just ~128MB567 // TODO: explore better strategies568 new_type = GGML_TYPE_Q8_0;569 }570 ++qs.i_attention_wv;571 } else if (category == tensor_category::ATTENTION_K) {572 if (qs.model.hparams.n_expert == 8) {573 // for the 8-expert model, bumping this to Q8_0 trades just ~128MB574 // TODO: explore better strategies575 new_type = GGML_TYPE_Q8_0;576 }577 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {578 new_type = GGML_TYPE_IQ3_XXS;579 }580 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {581 new_type = GGML_TYPE_IQ2_S;582 }583 } else if (category == tensor_category::ATTENTION_Q) {584 if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {585 new_type = GGML_TYPE_IQ3_XXS;586 }587 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {588 new_type = GGML_TYPE_IQ2_S;589 }590 } else if (category == tensor_category::FFN_DOWN) {591 auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str());592 int i_layer = info.first, n_layer = info.second;593 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;594 else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) {595 if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K;596 }597 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) {598 new_type = i_layer < n_layer/8 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;599 }600 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {601 new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K602 : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K603 : GGML_TYPE_Q3_K;604 }605 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M && (i_layer < n_layer/8 ||606 (qs.model.hparams.n_expert == 8 && use_more_bits(i_layer, n_layer)))) {607 new_type = GGML_TYPE_Q4_K;608 }609 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) {610 new_type = arch == LLM_ARCH_FALCON ? GGML_TYPE_Q4_K : GGML_TYPE_Q5_K;611 }612 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {613 if (arch == LLM_ARCH_FALCON) {614 new_type = i_layer < n_layer/16 ? GGML_TYPE_Q6_K :615 use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;616 } else {617 if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;618 }619 }620 else if (i_layer < n_layer/8 && (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && !qs.has_imatrix) {621 new_type = GGML_TYPE_Q5_K;622 }623 else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;624 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) {625 new_type = GGML_TYPE_Q5_K;626 }627 else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || ftype == LLAMA_FTYPE_MOSTLY_Q5_0)628 && qs.has_imatrix && i_layer < n_layer/8) {629 // Guard against craziness in the first few ffn_down layers that can happen even with imatrix for Q4_0/Q5_0.630 // We only do it when an imatrix is provided because a) we want to make sure that one can always get the631 // same quantization as before imatrix stuff, and b) Q4_1/Q5_1 do go crazy on ffn_down without an imatrix.632 new_type = ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q5_1;633 }634 ++qs.i_ffn_down;635 } else if (category == tensor_category::ATTENTION_OUTPUT) {636 if (arch != LLM_ARCH_FALCON) {637 if (qs.model.hparams.n_expert == 8) {638 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||639 ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL ||640 ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S ||641 ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) {642 new_type = GGML_TYPE_Q5_K;643 }644 } else {645 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K;646 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_IQ3_S;647 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ) new_type = GGML_TYPE_Q4_K;648 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ) new_type = GGML_TYPE_Q5_K;649 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M ) new_type = GGML_TYPE_Q4_K;650 }651 } else {652 if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K;653 }654 }655 else if (category == tensor_category::ATTENTION_QKV) {656 if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L || ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {657 new_type = GGML_TYPE_Q4_K;658 }659 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) new_type = GGML_TYPE_Q5_K;660 else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) new_type = GGML_TYPE_Q6_K;661 }662 else if (category == tensor_category::FFN_GATE) {663 auto info = layer_info(qs.i_ffn_gate, qs.n_ffn_gate, name.c_str());664 int i_layer = info.first, n_layer = info.second;665 if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {666 new_type = GGML_TYPE_IQ3_XXS;667 }668 ++qs.i_ffn_gate;669 }670 else if (category == tensor_category::FFN_UP) {671 auto info = layer_info(qs.i_ffn_up, qs.n_ffn_up, name.c_str());672 int i_layer = info.first, n_layer = info.second;673 if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {674 new_type = GGML_TYPE_IQ3_XXS;675 }676 ++qs.i_ffn_up;677 }678 679 return new_type;680}681 682// outer wrapper: determine the ggml_type that this tensor should be quantized to683static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_model_quantize_params * params, const ggml_tensor * tensor, ggml_type default_type, const tensor_metadata & tm) {684 if (!tensor_allows_quantization(params, qs.model.arch, tensor)) {685 return tensor->type;686 }687 if (params->token_embedding_type < GGML_TYPE_COUNT && tm.category == tensor_category::TOKEN_EMBD) {688 // per_layer_token_embd follows --token-embedding-type by default, but it is a large689 // separate table, so let an explicit --tensor-type name it690 bool named = false;691 if (std::strcmp(tensor->name, "per_layer_token_embd.weight") == 0) {692 const std::string tensor_name(tensor->name);693 for (const auto & [pattern, qtype] : qs.tensor_type_patterns) {694 if (std::regex_search(tensor_name, pattern)) {695 named = true;696 break;697 }698 }699 }700 if (!named) {701 return params->token_embedding_type;702 }703 }704 if (params->output_tensor_type < GGML_TYPE_COUNT && tm.category == tensor_category::OUTPUT) {705 return params->output_tensor_type;706 }707 708 ggml_type new_type = default_type;709 710 // get more optimal quantization type based on the tensor shape, layer, etc.711 if (ggml_is_quantized(default_type)) {712 // if the user provided tensor types - use those713 bool manual = false;714 if (!qs.tensor_type_patterns.empty()) {715 const std::string tensor_name(tensor->name);716 for (const auto & [pattern, qtype] : qs.tensor_type_patterns) {717 if (std::regex_search(tensor_name, pattern)) {718 if (qtype != new_type) {719 LLAMA_LOG_WARN("%s: %-36s - applying manual override: %s -> %s\n",720 __func__, tensor_name.c_str(), ggml_type_name(new_type), ggml_type_name(qtype));721 new_type = qtype;722 }723 manual = true;724 break;725 }726 }727 }728 729 // if not manual - use the standard logic for choosing the quantization type based on the selected mixture730 if (!manual && !params->pure) {731 new_type = llama_tensor_get_type_impl(qs, new_type, tensor, params->ftype, tm.category);732 }733 734 // incompatible tensor shapes are handled here - fallback to a compatible type735 new_type = tensor_type_fallback(qs, tensor, new_type);736 }737 738 return new_type;739}740 741//742// quantization implementation743//744 745// quantize rows [first_row, first_row + nrows), indexed globally across all expert matrices746// note: chunks never cross an expert boundary since each expert has its own imatrix slice747static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t first_row, int64_t nrows, int64_t nrows_per_expert, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {748 const size_t row_size = ggml_row_size(new_type, n_per_row);749 750 auto imatrix_for_row = [=](int64_t row_global) {751 return imatrix ? imatrix + (row_global / nrows_per_expert) * n_per_row : nullptr;752 };753 754 if (nthread < 2) {755 // single-thread756 size_t new_size = 0;757 for (int64_t row = 0; row < nrows;) {758 const int64_t row_global = first_row + row;759 const int64_t this_nrow = std::min(nrows - row, nrows_per_expert - row_global % nrows_per_expert);760 void * this_data = (char *) new_data + row * row_size;761 size_t this_size = ggml_quantize_chunk(new_type, f32_data + row * n_per_row, this_data, 0, this_nrow, n_per_row, imatrix_for_row(row_global));762 if (!ggml_validate_row_data(new_type, this_data, this_size)) {763 throw std::runtime_error("quantized data validation failed");764 }765 new_size += this_size;766 row += this_nrow;767 }768 return new_size;769 }770 771 std::mutex mutex;772 int64_t counter = 0;773 size_t new_size = 0;774 bool valid = true;775 auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,776 first_row, nrows, nrows_per_expert, n_per_row, row_size, imatrix_for_row]() {777 const int64_t nrows_per_chunk = chunk_size / n_per_row;778 size_t local_size = 0;779 while (true) {780 std::unique_lock<std::mutex> lock(mutex);781 if (counter >= nrows) {782 if (local_size > 0) {783 new_size += local_size;784 }785 break;786 }787 const int64_t row = counter;788 const int64_t row_global = first_row + row;789 // stop at the expert boundary790 const int64_t this_nrow = std::min(std::min(nrows - row, nrows_per_chunk), nrows_per_expert - row_global % nrows_per_expert);791 counter += this_nrow;792 lock.unlock();793 794 void * this_data = (char *) new_data + row * row_size;795 size_t this_size = ggml_quantize_chunk(new_type, f32_data + row * n_per_row, this_data, 0, this_nrow, n_per_row, imatrix_for_row(row_global));796 local_size += this_size;797 798 // validate the quantized data799 if (!ggml_validate_row_data(new_type, this_data, this_size)) {800 std::unique_lock<std::mutex> lock(mutex);801 valid = false;802 break;803 }804 }805 };806 for (int it = 0; it < nthread - 1; ++it) {807 workers.emplace_back(compute);808 }809 compute();810 for (auto & w : workers) { w.join(); }811 workers.clear();812 if (!valid) {813 throw std::runtime_error("quantized data validation failed");814 }815 return new_size;816}817 818//819// imatrix requirement check820//821 822static bool tensor_requires_imatrix(const char * tensor_name, const ggml_type dst_type, const llama_ftype ftype) {823 if (tensor_name_match_token_embd(tensor_name) || tensor_name_match_output_weight(tensor_name)) {824 return false;825 }826 switch (dst_type) {827 case GGML_TYPE_IQ3_XXS:828 case GGML_TYPE_IQ2_XXS:829 case GGML_TYPE_IQ2_XS:830 case GGML_TYPE_IQ2_S:831 case GGML_TYPE_IQ1_M:832 case GGML_TYPE_IQ1_S:833 return true;834 case GGML_TYPE_Q2_K:835 // as a general rule, the k-type quantizations don't require imatrix data.836 // the only exception is Q2_K tensors that are part of a Q2_K_S file.837 return ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S;838 default:839 return false;840 }841}842 843//844// given a file type, get the default tensor type845//846 847ggml_type llama_ftype_get_default_type(llama_ftype ftype) {848 switch (ftype) {849 case LLAMA_FTYPE_MOSTLY_Q4_0: return GGML_TYPE_Q4_0;850 case LLAMA_FTYPE_MOSTLY_Q4_1: return GGML_TYPE_Q4_1;851 case LLAMA_FTYPE_MOSTLY_Q5_0: return GGML_TYPE_Q5_0;852 case LLAMA_FTYPE_MOSTLY_Q5_1: return GGML_TYPE_Q5_1;853 case LLAMA_FTYPE_MOSTLY_Q8_0: return GGML_TYPE_Q8_0;854 case LLAMA_FTYPE_MOSTLY_F16: return GGML_TYPE_F16;855 case LLAMA_FTYPE_MOSTLY_BF16: return GGML_TYPE_BF16;856 case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32;857 case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0;858 case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0;859 860 case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4;861 862 // K-quants863 case LLAMA_FTYPE_MOSTLY_Q2_K_S:864 case LLAMA_FTYPE_MOSTLY_Q2_K: return GGML_TYPE_Q2_K;865 case LLAMA_FTYPE_MOSTLY_IQ3_XS: return GGML_TYPE_IQ3_S;866 case LLAMA_FTYPE_MOSTLY_Q3_K_S:867 case LLAMA_FTYPE_MOSTLY_Q3_K_M:868 case LLAMA_FTYPE_MOSTLY_Q3_K_L: return GGML_TYPE_Q3_K;869 case LLAMA_FTYPE_MOSTLY_Q4_K_S:870 case LLAMA_FTYPE_MOSTLY_Q4_K_M: return GGML_TYPE_Q4_K;871 case LLAMA_FTYPE_MOSTLY_Q5_K_S:872 case LLAMA_FTYPE_MOSTLY_Q5_K_M: return GGML_TYPE_Q5_K;873 case LLAMA_FTYPE_MOSTLY_Q6_K: return GGML_TYPE_Q6_K;874 case LLAMA_FTYPE_MOSTLY_TQ1_0: return GGML_TYPE_TQ1_0;875 case LLAMA_FTYPE_MOSTLY_TQ2_0: return GGML_TYPE_TQ2_0;876 case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return GGML_TYPE_IQ2_XXS;877 case LLAMA_FTYPE_MOSTLY_IQ2_XS: return GGML_TYPE_IQ2_XS;878 case LLAMA_FTYPE_MOSTLY_IQ2_S: return GGML_TYPE_IQ2_XS;879 case LLAMA_FTYPE_MOSTLY_IQ2_M: return GGML_TYPE_IQ2_S;880 case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return GGML_TYPE_IQ3_XXS;881 case LLAMA_FTYPE_MOSTLY_IQ1_S: return GGML_TYPE_IQ1_S;882 case LLAMA_FTYPE_MOSTLY_IQ1_M: return GGML_TYPE_IQ1_M;883 case LLAMA_FTYPE_MOSTLY_IQ4_NL: return GGML_TYPE_IQ4_NL;884 case LLAMA_FTYPE_MOSTLY_IQ4_XS: return GGML_TYPE_IQ4_XS;885 case LLAMA_FTYPE_MOSTLY_IQ3_S:886 case LLAMA_FTYPE_MOSTLY_IQ3_M: return GGML_TYPE_IQ3_S;887 888 default: return GGML_TYPE_COUNT;889 }890}891 892 893static void init_quantize_state_counters(quantize_state_impl & qs, std::vector<tensor_metadata> & metadata) {894 for (auto & tm : metadata) {895 tensor_category cat = tensor_get_category(tm.name);896 tm.category = cat;897 898 if (category_is_attn_v(cat)) {899 ++qs.n_attention_wv;900 }901 902 if (cat == tensor_category::OUTPUT) {903 qs.has_tied_embeddings = false;904 }905 }906 qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)qs.model.hparams.n_layer_all;907}908 909//910// main quantization driver911//912 913static void llama_model_quantize_impl(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) {914 llama_ftype ftype = params->ftype;915 916 int nthread = params->nthread;917 918 if (nthread <= 0) {919 nthread = std::thread::hardware_concurrency();920 }921 922 ggml_type default_type = llama_ftype_get_default_type(ftype);923 if (default_type == GGML_TYPE_COUNT) {924 throw std::runtime_error(format("invalid output file type %d\n", ftype));925 }926 927 // mmap consistently increases speed on Linux, and also increases speed on Windows with928 // hot cache. It may cause a slowdown on macOS, possibly related to free memory.929#if defined(__linux__) || defined(_WIN32)930 constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP;931#else932 constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_NONE;933#endif934 935 const llama_model_kv_override * kv_overrides = params->kv_overrides;936 std::vector<std::string> splits = {};937 llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr,938 fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, /*load_mtp*/ true, kv_overrides, nullptr);939 ml.init_mappings(false); // no prefetching940 941 auto mparams = llama_model_default_params();942 std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, mparams));943 944 auto * model = dynamic_cast<llama_model_base *>(model_ptr.get());945 if (model == nullptr) {946 GGML_ABORT("fatal error: model does not implement llama_model_base");947 }948 949 model->load_hparams(ml);950 model->load_stats (ml);951 952 quantize_state_impl qs(*model, params);953 954 if (params->only_copy) {955 ftype = ml.ftype;956 }957 std::unordered_map<std::string, std::vector<float>> i_data;958 const std::unordered_map<std::string, std::vector<float>> * imatrix_data = nullptr;959 if (params->imatrix) {960 for (const llama_model_imatrix_data * p = params->imatrix; p->name != nullptr; p++) {961 i_data.emplace(p->name, std::vector<float>(p->data, p->data + p->size));962 }963 imatrix_data = & i_data;964 if (imatrix_data) {965 LLAMA_LOG_INFO("\n%s: have importance matrix data with %d entries\n",966 __func__, (int)imatrix_data->size());967 qs.has_imatrix = true;968 // check imatrix for nans or infs969 for (const auto & kv : *imatrix_data) {970 for (float f : kv.second) {971 if (!std::isfinite(f)) {972 throw std::runtime_error(format("imatrix contains non-finite value %f\n", f));973 }974 }975 }976 }977 }978 979 const size_t align = GGUF_DEFAULT_ALIGNMENT;980 gguf_context_ptr ctx_out { gguf_init_empty() };981 982 std::vector<int> prune_list = {};983 if (params->prune_layers) {984 for (const int32_t * p = params->prune_layers; * p != -1; p++) {985 prune_list.push_back(* p);986 }987 }988 989 // copy the KV pairs from the input file990 gguf_set_kv (ctx_out.get(), ml.metadata);991 gguf_set_val_u32(ctx_out.get(), ml.llm_kv(LLM_KV_GENERAL_QUANTIZATION_VERSION).c_str(), GGML_QNT_VERSION);992 gguf_set_val_u32(ctx_out.get(), ml.llm_kv(LLM_KV_GENERAL_FILE_TYPE).c_str(), ftype);993 994 // Remove split metadata995 gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_NO).c_str());996 gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str());997 gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str());998 999 if (params->kv_overrides) {1000 for (const llama_model_kv_override * o = params->kv_overrides; o->key[0] != 0; ++o) {1001 if (o->tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {1002 gguf_set_val_f32(ctx_out.get(), o->key, o->val_f64);1003 } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_INT) {1004 // Setting type to UINT32. See https://github.com/ggml-org/llama.cpp/pull/14182 for context1005 gguf_set_val_u32(ctx_out.get(), o->key, (uint32_t)std::abs(o->val_i64));1006 } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {1007 gguf_set_val_bool(ctx_out.get(), o->key, o->val_bool);1008 } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {1009 gguf_set_val_str(ctx_out.get(), o->key, o->val_str);1010 } else {1011 LLAMA_LOG_WARN("%s: unknown KV override type for key %s\n", __func__, o->key);1012 }1013 }1014 }1015 1016 std::map<int, std::string> mapped;1017 int blk_id = 0;1018 1019 // make a list of weights1020 std::vector<const llama_model_loader::llama_tensor_weight *> tensors;1021 tensors.reserve(ml.weights_map.size());1022 for (const auto & it : ml.weights_map) {1023 const std::string remapped_name(remap_layer(it.first, prune_list, mapped, blk_id));1024 if (remapped_name.empty()) {1025 LLAMA_LOG_DEBUG("%s: pruning tensor %s\n", __func__, it.first.c_str());1026 continue;1027 }1028 1029 if (remapped_name != it.first) {1030 ggml_set_name(it.second.tensor, remapped_name.c_str());1031 LLAMA_LOG_DEBUG("%s: tensor %s remapped to %s\n", __func__, it.first.c_str(), ggml_get_name(it.second.tensor));1032 }1033 tensors.push_back(&it.second);1034 }1035 if (!prune_list.empty()) {1036 gguf_set_val_u32(ctx_out.get(), ml.llm_kv(LLM_KV_BLOCK_COUNT).c_str(), blk_id);1037 }1038 1039 // keep_split requires that the weights are sorted by split index1040 if (params->keep_split) {1041 std::sort(tensors.begin(), tensors.end(), [](const llama_model_loader::llama_tensor_weight * a, const llama_model_loader::llama_tensor_weight * b) {1042 if (a->idx == b->idx) {1043 return a->offs < b->offs;1044 }1045 return a->idx < b->idx;1046 });1047 }1048 1049 // compute tensor metadata once and cache it1050 std::vector<tensor_metadata> metadata(tensors.size());1051 for (size_t i = 0; i < tensors.size(); ++i) {1052 metadata[i].name = ggml_get_name(tensors[i]->tensor);1053 }1054 1055 // initialize quantization state counters and metadata categories1056 init_quantize_state_counters(qs, metadata);1057 1058 int idx = 0;1059 uint16_t n_split = 1;1060 1061 // Assume split index is continuous1062 if (params->keep_split) {1063 for (const auto * it : tensors) {1064 n_split = std::max(uint16_t(it->idx + 1), n_split);1065 }1066 }1067 std::vector<gguf_context_ptr> ctx_outs(n_split);1068 ctx_outs[0] = std::move(ctx_out);1069 1070 // flag for --dry-run1071 bool will_require_imatrix = false;1072 1073 //1074 // preliminary iteration over all weights1075 //1076 1077 for (size_t i = 0; i < tensors.size(); ++i) {1078 const auto * it = tensors[i];1079 const struct ggml_tensor * tensor = it->tensor;1080 1081 uint16_t i_split = params->keep_split ? it->idx : 0;1082 if (!ctx_outs[i_split]) {1083 ctx_outs[i_split].reset(gguf_init_empty());1084 }1085 gguf_add_tensor(ctx_outs[i_split].get(), tensor);1086 1087 metadata[i].allows_quantization = tensor_allows_quantization(params, model->arch, tensor);1088 1089 if (metadata[i].allows_quantization) {1090 metadata[i].target_type = llama_tensor_get_type(qs, params, tensor, default_type, metadata[i]);1091 } else {1092 metadata[i].target_type = tensor->type;1093 }1094 1095 metadata[i].requires_imatrix = tensor_requires_imatrix(tensor->name, metadata[i].target_type, ftype);1096 1097 if (params->imatrix) {1098 metadata[i].remapped_imatrix_name = remap_imatrix(tensor->name, mapped);1099 } else if (metadata[i].allows_quantization && metadata[i].requires_imatrix) {1100 if (params->dry_run) {1101 will_require_imatrix = true;1102 } else {1103 LLAMA_LOG_ERROR("\n============================================================================\n"1104 " ERROR: this quantization requires an importance matrix!\n"1105 " - offending tensor: %s\n"1106 " - target type: %s\n"1107 "============================================================================\n\n",1108 metadata[i].name.c_str(), ggml_type_name(metadata[i].target_type));1109 throw std::runtime_error("this quantization requires an imatrix!");1110 }1111 }1112 }1113 1114 // Set split info if needed1115 if (n_split > 1) {1116 for (size_t i = 0; i < ctx_outs.size(); ++i) {1117 gguf_set_val_u16(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_NO).c_str(), i);1118 gguf_set_val_u16(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str(), n_split);1119 gguf_set_val_i32(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str(), (int32_t)tensors.size());1120 }1121 }1122 1123 size_t total_size_org = 0;1124 size_t total_size_new = 0;1125 1126 std::vector<std::thread> workers;1127 workers.reserve(nthread);1128 1129 std::vector<no_init<uint8_t>> read_data;1130 std::vector<no_init<uint8_t>> work;1131 std::vector<no_init<float>> f32_conv_buf;1132 1133 const size_t max_buf_size = params->max_buf_size ? params->max_buf_size : LLAMA_QUANT_MAX_BUF_SIZE;1134 1135 int cur_split = -1;1136 std::ofstream fout;1137 auto close_ofstream = [&]() {1138 // Write metadata and close file handler1139 if (fout.is_open()) {1140 fout.seekp(0);1141 std::vector<uint8_t> data(gguf_get_meta_size(ctx_outs[cur_split].get()));1142 gguf_get_meta_data(ctx_outs[cur_split].get(), data.data());1143 fout.write((const char *) data.data(), data.size());1144 fout.close();1145 }1146 };1147 auto new_ofstream = [&](int index) {1148 cur_split = index;1149 GGML_ASSERT(ctx_outs[cur_split] && "Find uninitialized gguf_context");1150 std::string fname = fname_out;1151 if (params->keep_split) {1152 std::vector<char> split_path(llama_path_max(), 0);1153 llama_split_path(split_path.data(), split_path.size(), fname_out.c_str(), cur_split, n_split);1154 fname = std::string(split_path.data());1155 }1156 1157 fout = std::ofstream(fname, std::ios::binary);1158 fout.exceptions(std::ofstream::failbit); // fail fast on write errors1159 const size_t meta_size = gguf_get_meta_size(ctx_outs[cur_split].get());1160 // placeholder for the meta data1161 ::zeros(fout, meta_size);1162 };1163 1164 // no output file for --dry-run1165 if (!params->dry_run) {1166 new_ofstream(0);1167 }1168 1169 //1170 // main loop: iterate over all weights1171 //1172 1173 for (size_t i = 0; i < tensors.size(); ++i) {1174 const auto & weight = *tensors[i];1175 const auto & tm = metadata[i];1176 ggml_tensor * tensor = weight.tensor;1177 1178 if (!params->dry_run && (weight.idx != cur_split && params->keep_split)) {1179 close_ofstream();1180 new_ofstream(weight.idx);1181 }1182 1183 const size_t tensor_size = ggml_nbytes(tensor);1184 1185 // read a byte range of the current tensor1186 auto load_range = [&](size_t offs, size_t size) -> const void * {1187 if (!ml.use_mmap && read_data.size() < size) {1188 read_data.resize(size);1189 }1190 return ml.load_data_range(weight, offs, size, read_data.data());1191 };1192 1193 LLAMA_LOG_INFO("[%4d/%4d] %-36s - [%s], type = %6s, ",1194 ++idx, ml.n_tensors,1195 ggml_get_name(tensor),1196 llama_format_tensor_shape(tensor).c_str(),1197 ggml_type_name(tensor->type));1198 1199 const ggml_type cur_type = tensor->type;1200 const ggml_type new_type = tm.target_type;