Felipe97/llama-cpp-compiled
01.1k
1#include "llama-model-saver.h"2 3#include "ggml.h"4#include "gguf.h"5 6#include "llama-arch.h"7#include "llama.h"8#include "llama-hparams.h"9#include "llama-model.h"10#include "llama-vocab.h"11 12#include <cstdint>13#include <string>14#include <unordered_set>15 16bool llama_model_saver_supports_arch(llm_arch arch) {17 switch (arch) {18 case LLM_ARCH_GEMMA3N:19 case LLM_ARCH_BITNET:20 case LLM_ARCH_T5:21 case LLM_ARCH_APERTUS:22 case LLM_ARCH_STEP35:23 return false;24 default:25 return true;26 }27}28 29llama_model_saver::llama_model_saver(const struct llama_model * model) :30 gguf_ctx(gguf_init_empty()), gguf_ctx_owned(true), model(model), llm_kv(model->arch) {31 GGML_ASSERT(llama_model_saver_supports_arch(model->arch));32}33 34llama_model_saver::llama_model_saver(enum llm_arch arch, struct gguf_context * gguf_ctx) :35 gguf_ctx(gguf_ctx == nullptr ? gguf_init_empty() : gguf_ctx), gguf_ctx_owned(gguf_ctx == nullptr), model(nullptr), llm_kv(arch) {}36 37llama_model_saver::~llama_model_saver() {38 if (gguf_ctx_owned) {39 gguf_free(gguf_ctx);40 }41}42 43void llama_model_saver::add_kv(const enum llm_kv key, const uint32_t value) {44 gguf_set_val_u32(gguf_ctx, llm_kv(key).c_str(), value);45}46 47void llama_model_saver::add_kv(const enum llm_kv key, const int32_t value) {48 gguf_set_val_i32(gguf_ctx, llm_kv(key).c_str(), value);49}50 51void llama_model_saver::add_kv(const enum llm_kv key, const uint64_t value) {52 gguf_set_val_u64(gguf_ctx, llm_kv(key).c_str(), value);53}54 55void llama_model_saver::add_kv(const enum llm_kv key, const float value) {56 gguf_set_val_f32(gguf_ctx, llm_kv(key).c_str(), value);57}58 59void llama_model_saver::add_kv(const enum llm_kv key, const bool value) {60 gguf_set_val_bool(gguf_ctx, llm_kv(key).c_str(), value);61}62 63void llama_model_saver::add_kv(const enum llm_kv key, const char * value) {64 gguf_set_val_str(gguf_ctx, llm_kv(key).c_str(), value);65}66 67[[noreturn]]68void llama_model_saver::add_kv(const enum llm_kv key, const char value) {69 GGML_UNUSED(key);70 GGML_UNUSED(value);71 GGML_ABORT("fatal error"); // this should never be called, only needed to make the template below compile72}73 74template <typename Container>75void llama_model_saver::add_kv(const enum llm_kv key, const Container & value, const bool per_layer) {76 GGML_ASSERT(model != nullptr || !per_layer);77 const size_t n_values = per_layer ? size_t(model->hparams.n_layer()) : value.size();78 GGML_ASSERT(n_values <= value.size());79 80 if (n_values == 0) {81 return;82 }83 84 if (per_layer) {85 bool all_values_the_same = true;86 for (size_t i = 1; i < n_values; ++i) {87 if (value[i] != value[0]) {88 all_values_the_same = false;89 break;90 }91 }92 if (all_values_the_same) {93 add_kv(key, value[0]);94 return;95 }96 }97 98 if (std::is_same<typename Container::value_type, uint8_t>::value) {99 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_UINT8, value.data(), n_values);100 } else if (std::is_same<typename Container::value_type, int8_t>::value) {101 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_INT8, value.data(), n_values);102 } else if (std::is_same<typename Container::value_type, uint32_t>::value) {103 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_UINT32, value.data(), n_values);104 } else if (std::is_same<typename Container::value_type, bool>::value) {105 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_BOOL, value.data(), n_values);106 } else if (std::is_same<typename Container::value_type, int32_t>::value) {107 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_INT32, value.data(), n_values);108 } else if (std::is_same<typename Container::value_type, uint64_t>::value) {109 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_UINT64, value.data(), n_values);110 } else if (std::is_same<typename Container::value_type, float>::value) {111 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_FLOAT32, value.data(), n_values);112 } else if (std::is_same<Container, std::string>::value) {113 gguf_set_val_str(gguf_ctx, llm_kv(key).c_str(), reinterpret_cast<const char *>(value.data()));114 } else {115 GGML_ABORT("fatal error");116 }117}118// instantiate for external usage:119template void llama_model_saver::add_kv<std::vector<uint32_t>>(const enum llm_kv, const std::vector<uint32_t> &, const bool);120template void llama_model_saver::add_kv<std::vector<float>>(const enum llm_kv, const std::vector<float> &, const bool);121template void llama_model_saver::add_kv<std::vector<uint64_t>>(const enum llm_kv, const std::vector<uint64_t> &, const bool);122 123void llama_model_saver::add_kv(const enum llm_kv key, const std::vector<std::string> & value) {124 std::vector<const char *> tmp(value.size());125 for (size_t i = 0; i < value.size(); ++i) {126 tmp[i] = value[i].c_str();127 }128 gguf_set_arr_str(gguf_ctx, llm_kv(key).c_str(), tmp.data(), tmp.size());129}130 131void llama_model_saver::add_tensor(const struct ggml_tensor * tensor) {132 if (!tensor) {133 return;134 }135 if (gguf_find_tensor(gguf_ctx, tensor->name) >= 0) {136 const std::string tensor_name = tensor->name;137 GGML_ASSERT(138 tensor_name == "rope_freqs.weight" || tensor_name == "rope_factors_long.weight" ||139 tensor_name == "rope_factors_short.weight"); // FIXME140 return;141 }142 gguf_add_tensor(gguf_ctx, tensor);143}144 145void llama_model_saver::add_kv_from_model() {146 const llama_hparams & hparams = model->hparams;147 const llama_vocab & vocab = model->vocab;148 149 const int32_t n_vocab = vocab.n_tokens();150 std::vector<std::string> tokens(n_vocab);151 std::vector<float> scores(n_vocab);152 std::vector<int32_t> token_types(n_vocab);153 154 if (vocab.get_type() != LLAMA_VOCAB_TYPE_NONE) {155 for (int32_t id = 0; id < n_vocab; ++id) {156 const llama_vocab::token_data & token_data = vocab.get_token_data(id);157 158 tokens[id] = token_data.text;159 scores[id] = token_data.score;160 161 // FIXME should this be treated as flags?162 switch(token_data.attr) {163 case LLAMA_TOKEN_ATTR_UNKNOWN: token_types[id] = LLAMA_TOKEN_TYPE_UNKNOWN; break;164 case LLAMA_TOKEN_ATTR_UNUSED: token_types[id] = LLAMA_TOKEN_TYPE_UNUSED; break;165 case LLAMA_TOKEN_ATTR_NORMAL: token_types[id] = LLAMA_TOKEN_TYPE_NORMAL; break;166 case LLAMA_TOKEN_ATTR_CONTROL: token_types[id] = LLAMA_TOKEN_TYPE_CONTROL; break;167 case LLAMA_TOKEN_ATTR_USER_DEFINED: token_types[id] = LLAMA_TOKEN_TYPE_USER_DEFINED; break;168 case LLAMA_TOKEN_ATTR_BYTE: token_types[id] = LLAMA_TOKEN_TYPE_BYTE; break;169 // case LLAMA_TOKEN_ATTR_NORMALIZED: ???170 // case LLAMA_TOKEN_ATTR_LSTRIP: ???171 // case LLAMA_TOKEN_ATTR_RSTRIP: ???172 case LLAMA_TOKEN_ATTR_UNDEFINED:173 default: token_types[id] = LLAMA_TOKEN_TYPE_UNDEFINED; break;174 }175 }176 }177 178 // add_kv(LLM_KV_GENERAL_TYPE, ???);179 add_kv(LLM_KV_GENERAL_ARCHITECTURE, model->arch_name());180 // add_kv(LLM_KV_GENERAL_QUANTIZATION_VERSION, ???);181 // add_kv(LLM_KV_GENERAL_ALIGNMENT, ???);182 // add_kv(LLM_KV_GENERAL_FILE_TYPE, ???);183 // add_kv(LLM_KV_GENERAL_SAMPLING_SEQUENCE, ???);184 // add_kv(LLM_KV_GENERAL_SAMPLING_TOP_K, ???);185 // add_kv(LLM_KV_GENERAL_SAMPLING_TOP_P, ???);186 // add_kv(LLM_KV_GENERAL_SAMPLING_MIN_P, ???);187 // add_kv(LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY, ???);188 // add_kv(LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD, ???);189 // add_kv(LLM_KV_GENERAL_SAMPLING_TEMP, ???);190 // add_kv(LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N, ???);191 // add_kv(LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT, ???);192 // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT, ???);193 // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU, ???);194 // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA, ???);195 add_kv(LLM_KV_GENERAL_NAME, model->name);196 // add_kv(LLM_KV_GENERAL_AUTHOR, ???);197 // add_kv(LLM_KV_GENERAL_VERSION, ???);198 // add_kv(LLM_KV_GENERAL_URL, ???);199 // add_kv(LLM_KV_GENERAL_DESCRIPTION, ???);200 // add_kv(LLM_KV_GENERAL_LICENSE, ???);201 // add_kv(LLM_KV_GENERAL_SOURCE_URL, ???);202 // add_kv(LLM_KV_GENERAL_SOURCE_HF_REPO, ???);203 204 add_kv(LLM_KV_VOCAB_SIZE, vocab.n_tokens());205 add_kv(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);206 add_kv(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);207 if (hparams.n_embd_out_impl > 0) {208 add_kv(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl);209 }210 add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);211 add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);212 add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true);213 add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp());214 add_kv(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent);215 add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);216 add_kv(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);217 add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>(218 hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.begin() + hparams.n_layer_all));219 add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>(220 hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.begin() + hparams.n_layer_all));221 add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);222 // add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???);223 add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert);224 add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used());225 add_kv(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);226 add_kv(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups);227 add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used);228 add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);229 add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);230 add_kv(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);231 add_kv(LLM_KV_EXPERT_GROUP_SCALE, hparams.expert_group_scale);232 add_kv(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts);233 add_kv(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers);234 add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn);235 add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers);236 add_kv(LLM_KV_DEEPSTACK_MAPPING, hparams.deepstack_mapping_arr);237 add_kv(LLM_KV_POOLING_TYPE, uint32_t(hparams.pooling_type));238 add_kv(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);239 add_kv(LLM_KV_DECODER_START_TOKEN_ID, hparams.dec_start_token_id);240 add_kv(LLM_KV_DECODER_BLOCK_COUNT, hparams.dec_n_layer);241 add_kv(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping);242 add_kv(LLM_KV_ROUTER_LOGIT_SOFTCAPPING, hparams.f_router_logit_softcapping);243 add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);244 add_kv(LLM_KV_SWIN_NORM, hparams.swin_norm);245 add_kv(LLM_KV_RESCALE_EVERY_N_LAYERS, hparams.rescale_every_n_layers);246 add_kv(LLM_KV_TIME_MIX_EXTRA_DIM, hparams.time_mix_extra_dim);247 add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, hparams.time_decay_extra_dim);248 add_kv(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale);249 add_kv(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale);250 add_kv(LLM_KV_HRM_LAYERS_PER_STACK, hparams.n_hrm_layers_per_stack);251 add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles);252 add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles);253 add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm);254 add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count);255 add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);256 // add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???); // saved as LLM_KV_ATTENTION_RECURRENT_LAYERS instead257 258 add_kv(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, true);259 add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, true);260 add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);261 add_kv(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);262 add_kv(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k_full);263 add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full);264 add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);265 add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);266 add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, hparams.f_norm_group_eps);267 add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);268 add_kv(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);269 add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);270 add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);271 add_kv(LLM_KV_ATTENTION_DECAY_LORA_RANK, hparams.n_lora_decay);272 add_kv(LLM_KV_ATTENTION_ICLR_LORA_RANK, hparams.n_lora_iclr);273 add_kv(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix);274 add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate);275 add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);276 add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, hparams.rope_pattern, true);277 add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);278 if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {279 // never collapsed to a scalar: the loaders read a scalar as a period280 add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>(281 hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all));282 }283 add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);284 add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale);285 add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale);286 add_kv(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length);287 add_kv(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale);288 add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);289 add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);290 add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa);291 add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa);292 add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, hparams.n_embd_head_k_mla_swa);293 add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, hparams.n_embd_head_v_mla_swa);294 add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, hparams.n_lora_kv_swa);295 add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);296 add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);297 add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);298 add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size);299 add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks);300 add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, true);301 add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true);302 add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count);303 add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank);304 add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base);305 if (model->arch == LLM_ARCH_DEEPSEEK4 || hparams.dsv4_hc_mult > 0) {306 // the loader requires one compress ratio per layer, including nextn layers307 const std::vector<uint32_t> compress_ratios(308 hparams.dsv4_compress_ratios.begin(), hparams.dsv4_compress_ratios.begin() + hparams.n_layer_all);309 add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, compress_ratios);310 } else {311 add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, true);312 }313 add_kv(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);314 add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters);315 add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps);316 add_kv(LLM_KV_HYPER_CONNECTION_MAGNITUDE, hparams.hc_magnitude);317 add_kv(LLM_KV_HASH_LAYER_COUNT, hparams.dsv4_hash_layer_count);318 add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);319 320 // the PLE group only means anything whole: write all of it or none321 if (hparams.ple_n_heads > 0) {322 std::vector<uint32_t> ple_layers;323 for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {324 if (hparams.is_ple_impl[il]) {325 ple_layers.push_back(il);326 }327 }328 add_kv(LLM_KV_PLE_LAYERS, ple_layers);329 add_kv(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size);330 add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram);331 add_kv(LLM_KV_PLE_CONV_KERNEL, hparams.ple_conv_kernel);332 add_kv(LLM_KV_PLE_EOS_TOKEN_ID, hparams.ple_eos_token_id);333 add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.ple_head_dim);334 add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>(335 hparams.ple_layer_multipliers.begin(),336 hparams.ple_layer_multipliers.begin() + hparams.ple_ngram_size));337 add_kv(LLM_KV_PLE_HEAD_OFFSETS, std::vector<uint64_t>(338 hparams.ple_head_offsets.begin(),339 hparams.ple_head_offsets.begin() + hparams.ple_n_heads));340 add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, std::vector<uint64_t>(341 hparams.ple_head_vocab_sizes.begin(),342 hparams.ple_head_vocab_sizes.begin() + hparams.ple_n_heads));343 }344 345 const float rope_scaling_factor = hparams.rope_freq_scale_train == 1.0f ? 0.0f : 1.0f/hparams.rope_freq_scale_train;346 347 add_kv(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot_full);348 add_kv(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa);349 add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections);350 add_kv(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train);351 add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);352 // add_kv(LLM_KV_ROPE_SCALE_LINEAR, rope_scaling_factor); // old name353 add_kv(LLM_KV_ROPE_SCALING_TYPE, llama_rope_scaling_type_name(hparams.rope_scaling_type_train));354 add_kv(LLM_KV_ROPE_SCALING_FACTOR, rope_scaling_factor);355 add_kv(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor);356 add_kv(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn);357 add_kv(LLM_KV_ROPE_SCALING_FINETUNED, hparams.rope_finetuned);358 add_kv(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul);359 add_kv(LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR, hparams.yarn_ext_factor);360 add_kv(LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR, hparams.yarn_attn_factor);361 add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_FAST, hparams.yarn_beta_fast);362 add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, hparams.yarn_beta_slow);363 364 // TODO: implement split file support365 // add_kv(LLM_KV_SPLIT_NO, ???);366 // add_kv(LLM_KV_SPLIT_COUNT, ???);367 // add_kv(LLM_KV_SPLIT_TENSORS_COUNT, ???);368 369 add_kv(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);370 add_kv(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);371 add_kv(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);372 add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);373 add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);374 add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);375 376 add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);377 add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate);378 add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound);379 380 add_kv(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);381 382 add_kv(LLM_KV_TOKENIZER_MODEL, vocab.get_tokenizer_model());383 add_kv(LLM_KV_TOKENIZER_PRE, vocab.get_tokenizer_pre());384 add_kv(LLM_KV_TOKENIZER_LIST, tokens);385 add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE, token_types);386 add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, vocab.n_token_types());387 add_kv(LLM_KV_TOKENIZER_SCORES, scores);388 add_kv(LLM_KV_TOKENIZER_MERGES, vocab.get_bpe_merges());389 // FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though390 if (vocab.token_bos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); }391 if (vocab.token_eos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); }392 if (vocab.token_eot() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot())); }393 if (vocab.token_eom() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom())); }394 if (vocab.token_unk() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk())); }395 if (vocab.token_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep())); }396 if (vocab.token_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad())); }397 // add_kv(LLM_KV_TOKENIZER_CLS_ID, uint32_t(vocab.token_bos())); // deprecated398 // add_kv(LLM_KV_TOKENIZER_MASK_ID, ???);399 add_kv(LLM_KV_TOKENIZER_ADD_BOS, vocab.get_add_bos());400 add_kv(LLM_KV_TOKENIZER_ADD_EOS, vocab.get_add_eos());401 add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep());402 add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix());403 add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces());404 add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());405 // add_kv(LLM_KV_TOKENIZER_HF_JSON, ???);406 // add_kv(LLM_KV_TOKENIZER_RWKV, ???);407 if (vocab.token_fim_pre() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre())); }408 if (vocab.token_fim_suf() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf())); }409 if (vocab.token_fim_mid() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid())); }410 if (vocab.token_fim_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad())); }411 if (vocab.token_fim_rep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep())); }412 if (vocab.token_fim_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep())); }413 414 // TODO: implement LoRA support415 // add_kv(LLM_KV_ADAPTER_TYPE, ???);416 // add_kv(LLM_KV_ADAPTER_LORA_ALPHA, ???);417 // add_kv(LLM_KV_ADAPTER_LORA_TASK_NAME, ???);418 // add_kv(LLM_KV_ADAPTER_LORA_PROMPT_PREFIX, ???);419 // add_kv(LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS, ???);420 421 add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);422 add_kv(LLM_KV_POSNET_BLOCK_COUNT, hparams.posnet.n_layer);423 424 add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);425 add_kv(LLM_KV_CONVNEXT_BLOCK_COUNT, hparams.convnext.n_layer);426 427 add_kv(LLM_KV_CLASSIFIER_OUTPUT_LABELS, model->classifier_labels);428 429 add_kv(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);430 431 add_kv(LLM_KV_XIELU_ALPHA_N, hparams.xielu_alpha_n);432 add_kv(LLM_KV_XIELU_ALPHA_P, hparams.xielu_alpha_p);433 add_kv(LLM_KV_XIELU_BETA, hparams.xielu_beta);434 add_kv(LLM_KV_XIELU_EPS, hparams.xielu_eps);435 436 add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);437 add_kv(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);438 add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);439 440 // deprecated441 // add_kv(LLM_KV_TOKENIZER_PREFIX_ID, ???);442 // add_kv(LLM_KV_TOKENIZER_SUFFIX_ID, ???);443 // add_kv(LLM_KV_TOKENIZER_MIDDLE_ID, ???);444 445 add_kv(LLM_KV_DENSE_2_FEAT_IN, hparams.dense_2_feat_in);446 add_kv(LLM_KV_DENSE_2_FEAT_OUT, hparams.dense_2_feat_out);447 add_kv(LLM_KV_DENSE_3_FEAT_IN, hparams.dense_3_feat_in);448 add_kv(LLM_KV_DENSE_3_FEAT_OUT, hparams.dense_3_feat_out);449}450 451void llama_model_saver::add_tensors_from_model() {452 if (model->output != nullptr &&453 std::string(model->output->name) != std::string(model->tok_embd->name)) {454 add_tensor(model->tok_embd); // some models use the same tensor for tok_embd and output455 }456 add_tensor(model->type_embd);457 add_tensor(model->pos_embd);458 add_tensor(model->tok_norm);459 add_tensor(model->tok_norm_b);460 add_tensor(model->output_norm);461 add_tensor(model->output_norm_b);462 add_tensor(model->output);463 add_tensor(model->output_b);464 add_tensor(model->output_norm_enc);465 add_tensor(model->output_s);466 add_tensor(model->output_in_s);467 add_tensor(model->output_res_score);468 add_tensor(model->nextn_proj_pre);469 add_tensor(model->nextn_proj_post);470 add_tensor(model->cls);471 add_tensor(model->cls_b);472 add_tensor(model->cls_out);473 add_tensor(model->cls_out_b);474 add_tensor(model->cls_norm);475 add_tensor(model->hrm_z_l_init);476 add_tensor(model->hc_head_fn);477 add_tensor(model->hc_head_base);478 add_tensor(model->hc_head_scale);479 add_tensor(model->per_layer_tok_embd);480 add_tensor(model->hc_head_norm);481 add_tensor(model->hc_head_down);482 add_tensor(model->hc_head_up);483 484 // looped architectures alias physical tensors across cache slots; save each485 // tensor once. a different tensor with an existing name still asserts below486 std::unordered_set<const struct ggml_tensor *> seen;487 488 for (const struct llama_layer & layer : model->layers) {489 for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) {490 const struct ggml_tensor * tensor = reinterpret_cast<const struct ggml_tensor * const *>(&layer)[i];491 if (tensor == nullptr || !seen.insert(tensor).second) {492 continue;493 }494 add_tensor(tensor);495 }496 }497}498 499void llama_model_saver::save(const std::string & path_model) {500 gguf_write_to_file(gguf_ctx, path_model.c_str(), false);501}502 503void llama_model_saver::save(FILE * file) {504 gguf_write_to_file_ptr(gguf_ctx, file, false);505}506