sinimiini/HRM-Text-1B-GGUF
21268
1diff --git a/conversion/__init__.py b/conversion/__init__.py2index 2c38123df..ecf1be2db 1006443--- a/conversion/__init__.py4+++ b/conversion/__init__.py5@@ -95,6 +95,7 @@ TEXT_MODEL_MAP: dict[str, str] = {6 "HunYuanDenseV1ForCausalLM": "hunyuan",7 "HunYuanMoEV1ForCausalLM": "hunyuan",8 "HunYuanVLForConditionalGeneration": "hunyuan",9+ "HrmTextForCausalLM": "hrm_text",10 "IQuestCoderForCausalLM": "llama",11 "InternLM2ForCausalLM": "internlm",12 "InternLM3ForCausalLM": "internlm",13diff --git a/conversion/hrm_text.py b/conversion/hrm_text.py14new file mode 10064415index 000000000..1f29ab55e16--- /dev/null17+++ b/conversion/hrm_text.py18@@ -0,0 +1,120 @@19+from __future__ import annotations20+21+import re22+import json23+24+from typing import Iterable, TYPE_CHECKING25+26+import torch27+28+if TYPE_CHECKING:29+ from torch import Tensor30+31+from .base import ModelBase, TextModel, gguf, logger32+33+34+@ModelBase.register("HrmTextForCausalLM")35+class HrmTextModel(TextModel):36+ model_arch = gguf.MODEL_ARCH.HRM_TEXT37+38+ def __init__(self, *args, **kwargs):39+ super().__init__(*args, **kwargs)40+41+ with open(self.dir_model / "config.json", "r", encoding="utf-8") as f:42+ self.raw_hparams = json.load(f)43+44+ self.layers_per_stack = self.raw_hparams["num_hidden_layers"]45+ self.h_cycles = self.raw_hparams["H_cycles"]46+ self.l_cycles = self.raw_hparams["L_cycles"]47+ self.physical_block_count = self.layers_per_stack * 248+ self.cache_block_count = self.layers_per_stack * self.h_cycles * (self.l_cycles + 1)49+50+ # GGUF tensors store one physical L stack followed by one physical H stack.51+ # The runtime expands these 32 physical layers across 128 KV-cache slots.52+ self.block_count = self.physical_block_count53+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)54+55+ def set_vocab(self):56+ # HRM-Text ships a Qwen2-style tokenizer.json. Keep it as a plain tokenizer;57+ # do not add a chat template for validation GGUFs.58+ self._set_vocab_gpt2()59+60+ def get_vocab_base_pre(self, tokenizer) -> str:61+ del tokenizer62+ return "qwen2"63+64+ def set_gguf_parameters(self):65+ hp = self.raw_hparams66+ head_dim = hp["head_dim"]67+68+ self.gguf_writer.add_context_length(hp["max_position_embeddings"])69+ self.gguf_writer.add_embedding_length(hp["hidden_size"])70+ self.gguf_writer.add_block_count(self.cache_block_count)71+ self.gguf_writer.add_feed_forward_length(hp["intermediate_size"])72+ self.gguf_writer.add_head_count(hp["num_attention_heads"])73+ self.gguf_writer.add_head_count_kv(hp["num_key_value_heads"])74+ self.gguf_writer.add_key_length(head_dim)75+ self.gguf_writer.add_value_length(head_dim)76+ self.gguf_writer.add_rope_dimension_count(head_dim)77+ self.gguf_writer.add_rope_freq_base(hp.get("rope_theta", 10000.0))78+ self.gguf_writer.add_layer_norm_rms_eps(hp["rms_norm_eps"])79+ self.gguf_writer.add_embedding_scale(hp["embedding_scale"])80+81+ arch = self.gguf_writer.arch82+ self.gguf_writer.add_uint32(gguf.Keys.LLM.HRM_LAYERS_PER_STACK.format(arch=arch), self.layers_per_stack)83+ self.gguf_writer.add_uint32(gguf.Keys.LLM.HRM_H_CYCLES.format(arch=arch), self.h_cycles)84+ self.gguf_writer.add_uint32(gguf.Keys.LLM.HRM_L_CYCLES.format(arch=arch), self.l_cycles)85+ self.gguf_writer.add_bool(gguf.Keys.LLM.HRM_PREFIX_LM.format(arch=arch), bool(hp.get("prefix_lm", False)))86+87+ def _format(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:88+ return self.format_tensor_name(key, bid=bid, suffix=suffix)89+90+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:91+ if name == "model.embed_tokens.weight":92+ yield self._format(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch93+ return94+95+ if name == "lm_head.weight":96+ yield self._format(gguf.MODEL_TENSOR.OUTPUT), data_torch97+ return98+99+ if name == "model.z_L_init":100+ yield self._format(gguf.MODEL_TENSOR.HRM_Z_L_INIT, suffix=""), data_torch101+ return102+103+ match = re.fullmatch(r"model\.([LH])_module\.layers\.(\d+)\.(.+)", name)104+ if match is None:105+ raise ValueError(f"Can not map tensor {name!r}")106+107+ stack, layer_s, tensor_name = match.groups()108+ layer_idx = int(layer_s)109+ if layer_idx >= self.layers_per_stack:110+ raise ValueError(f"Layer index {layer_idx} outside HRM stack size {self.layers_per_stack}")111+112+ physical_bid = layer_idx + (self.layers_per_stack if stack == "H" else 0)113+114+ if tensor_name == "attn.gqkv_proj.weight":115+ gate, q, k, v = torch.chunk(data_torch, 4, dim=0)116+ logger.debug("Split %s as gate, q, k, v", name)117+ yield self._format(gguf.MODEL_TENSOR.ATTN_GATE, physical_bid), gate.contiguous()118+ yield self._format(gguf.MODEL_TENSOR.ATTN_Q, physical_bid), q.contiguous()119+ yield self._format(gguf.MODEL_TENSOR.ATTN_K, physical_bid), k.contiguous()120+ yield self._format(gguf.MODEL_TENSOR.ATTN_V, physical_bid), v.contiguous()121+ return122+123+ if tensor_name == "attn.o_proj.weight":124+ yield self._format(gguf.MODEL_TENSOR.ATTN_OUT, physical_bid), data_torch125+ return126+127+ if tensor_name == "mlp.gate_up_proj.weight":128+ gate, up = torch.chunk(data_torch, 2, dim=0)129+ logger.debug("Split %s as gate, up", name)130+ yield self._format(gguf.MODEL_TENSOR.FFN_GATE, physical_bid), gate.contiguous()131+ yield self._format(gguf.MODEL_TENSOR.FFN_UP, physical_bid), up.contiguous()132+ return133+134+ if tensor_name == "mlp.down_proj.weight":135+ yield self._format(gguf.MODEL_TENSOR.FFN_DOWN, physical_bid), data_torch136+ return137+138+ raise ValueError(f"Can not map tensor {name!r}")139diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py140index 7fdcf03d7..b84cc8827 100644141--- a/gguf-py/gguf/constants.py142+++ b/gguf-py/gguf/constants.py143@@ -144,6 +144,10 @@ class Keys:144 TOKEN_SHIFT_COUNT = "{arch}.token_shift_count"145 INTERLEAVE_MOE_LAYER_STEP = "{arch}.interleave_moe_layer_step"146 FULL_ATTENTION_INTERVAL = "{arch}.full_attention_interval"147+ HRM_LAYERS_PER_STACK = "{arch}.layers_per_stack"148+ HRM_H_CYCLES = "{arch}.h_cycles"149+ HRM_L_CYCLES = "{arch}.l_cycles"150+ HRM_PREFIX_LM = "{arch}.prefix_lm"151 ACTIVATION_SPARSITY_SCALE = "{arch}.activation_sparsity_scale"152 ALTUP_ACTIVE_IDX = "{arch}.altup.active_idx"153 ALTUP_NUM_INPUTS = "{arch}.altup.num_inputs"154@@ -410,6 +414,7 @@ class MODEL_ARCH(IntEnum):155 QWEN3 = auto()156 QWEN3MOE = auto()157 QWEN3NEXT = auto()158+ HRM_TEXT = auto()159 QWEN3VL = auto()160 QWEN3VLMOE = auto()161 QWEN35 = auto()162@@ -527,6 +532,7 @@ class MODEL_TENSOR(IntEnum):163 TOKEN_TYPES = auto()164 POS_EMBD = auto()165 OUTPUT = auto()166+ HRM_Z_L_INIT = auto()167 DENSE_2_OUT = auto() # embeddinggemma 2_Dense168 DENSE_3_OUT = auto() # embeddinggemma 3_Dense169 OUTPUT_NORM = auto()170@@ -925,6 +931,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {171 MODEL_ARCH.QWEN3: "qwen3",172 MODEL_ARCH.QWEN3MOE: "qwen3moe",173 MODEL_ARCH.QWEN3NEXT: "qwen3next",174+ MODEL_ARCH.HRM_TEXT: "hrm_text",175 MODEL_ARCH.QWEN3VL: "qwen3vl",176 MODEL_ARCH.QWEN3VLMOE: "qwen3vlmoe",177 MODEL_ARCH.QWEN35: "qwen35",178@@ -1042,6 +1049,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {179 MODEL_TENSOR.POS_EMBD: "position_embd",180 MODEL_TENSOR.OUTPUT_NORM: "output_norm",181 MODEL_TENSOR.OUTPUT: "output",182+ MODEL_TENSOR.HRM_Z_L_INIT: "hrm.z_l_init",183 MODEL_TENSOR.DENSE_2_OUT: "dense_2", # embeddinggemma 2_Dense184 MODEL_TENSOR.DENSE_3_OUT: "dense_3", # embeddinggemma 2_Dense185 MODEL_TENSOR.ROPE_FREQS: "rope_freqs",186@@ -2057,6 +2065,19 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {187 MODEL_TENSOR.SSM_BETA_ALPHA,188 MODEL_TENSOR.SSM_OUT189 ],190+ MODEL_ARCH.HRM_TEXT: [191+ MODEL_TENSOR.TOKEN_EMBD,192+ MODEL_TENSOR.OUTPUT,193+ MODEL_TENSOR.HRM_Z_L_INIT,194+ MODEL_TENSOR.ATTN_Q,195+ MODEL_TENSOR.ATTN_K,196+ MODEL_TENSOR.ATTN_V,197+ MODEL_TENSOR.ATTN_GATE,198+ MODEL_TENSOR.ATTN_OUT,199+ MODEL_TENSOR.FFN_GATE,200+ MODEL_TENSOR.FFN_DOWN,201+ MODEL_TENSOR.FFN_UP,202+ ],203 MODEL_ARCH.QWEN3VL: [204 MODEL_TENSOR.TOKEN_EMBD,205 MODEL_TENSOR.OUTPUT_NORM,206diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp207index c9eead18a..5b8ee3781 100644208--- a/src/llama-arch.cpp209+++ b/src/llama-arch.cpp210@@ -37,6 +37,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {211 { LLM_ARCH_QWEN3, "qwen3" },212 { LLM_ARCH_QWEN3MOE, "qwen3moe" },213 { LLM_ARCH_QWEN3NEXT, "qwen3next" },214+ { LLM_ARCH_HRM_TEXT, "hrm_text" },215 { LLM_ARCH_QWEN3VL, "qwen3vl" },216 { LLM_ARCH_QWEN3VLMOE, "qwen3vlmoe" },217 { LLM_ARCH_QWEN35, "qwen35" },218@@ -209,6 +210,10 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {219 { LLM_KV_TOKEN_SHIFT_COUNT, "%s.token_shift_count" },220 { LLM_KV_INTERLEAVE_MOE_LAYER_STEP, "%s.interleave_moe_layer_step" },221 { LLM_KV_FULL_ATTENTION_INTERVAL, "%s.full_attention_interval" },222+ { LLM_KV_HRM_LAYERS_PER_STACK, "%s.layers_per_stack" },223+ { LLM_KV_HRM_H_CYCLES, "%s.h_cycles" },224+ { LLM_KV_HRM_L_CYCLES, "%s.l_cycles" },225+ { LLM_KV_HRM_PREFIX_LM, "%s.prefix_lm" },226 227 { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },228 { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },229@@ -346,6 +351,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {230 { LLM_TENSOR_OUTPUT_NORM, "output_norm" },231 { LLM_TENSOR_OUTPUT_NORM_LFM2, "token_embd_norm" }, // fix for wrong tensor name232 { LLM_TENSOR_OUTPUT, "output" },233+ { LLM_TENSOR_HRM_Z_L_INIT, "hrm.z_l_init" },234 { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },235 { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },236 { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },237@@ -565,6 +571,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {238 {LLM_TENSOR_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},239 {LLM_TENSOR_TOKEN_TYPES, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},240 {LLM_TENSOR_TOKEN_EMBD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, // do the norms on the first layer (not the input layer)241+ {LLM_TENSOR_HRM_Z_L_INIT, {LLM_TENSOR_LAYER_INPUT, GGML_OP_MUL}},242 {LLM_TENSOR_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},243 {LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},244 {LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},245diff --git a/src/llama-arch.h b/src/llama-arch.h246index 89cf16cc3..fa04b684b 100644247--- a/src/llama-arch.h248+++ b/src/llama-arch.h249@@ -41,6 +41,7 @@ enum llm_arch {250 LLM_ARCH_QWEN3,251 LLM_ARCH_QWEN3MOE,252 LLM_ARCH_QWEN3NEXT,253+ LLM_ARCH_HRM_TEXT,254 LLM_ARCH_QWEN3VL,255 LLM_ARCH_QWEN3VLMOE,256 LLM_ARCH_QWEN35,257@@ -213,6 +214,10 @@ enum llm_kv {258 LLM_KV_TOKEN_SHIFT_COUNT,259 LLM_KV_INTERLEAVE_MOE_LAYER_STEP,260 LLM_KV_FULL_ATTENTION_INTERVAL,261+ LLM_KV_HRM_LAYERS_PER_STACK,262+ LLM_KV_HRM_H_CYCLES,263+ LLM_KV_HRM_L_CYCLES,264+ LLM_KV_HRM_PREFIX_LM,265 266 LLM_KV_ATTENTION_HEAD_COUNT,267 LLM_KV_ATTENTION_HEAD_COUNT_KV,268@@ -354,6 +359,7 @@ enum llm_tensor {269 LLM_TENSOR_DENSE_2_OUT,270 LLM_TENSOR_DENSE_3_OUT,271 LLM_TENSOR_OUTPUT,272+ LLM_TENSOR_HRM_Z_L_INIT,273 LLM_TENSOR_OUTPUT_NORM,274 LLM_TENSOR_OUTPUT_NORM_LFM2, // fix for wrong tensor name275 LLM_TENSOR_ROPE_FREQS,276diff --git a/src/llama-context.cpp b/src/llama-context.cpp277index ad36c0666..fa80f4260 100644278--- a/src/llama-context.cpp279+++ b/src/llama-context.cpp280@@ -2208,6 +2208,9 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {281 if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE) {282 return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());283 }284+ if (model.arch == LLM_ARCH_HRM_TEXT) {285+ return std::max<uint32_t>(n_tokens * 80, 64u * model.n_tensors());286+ }287 uint32_t res = std::max<uint32_t>(1024u, 8u*model.n_tensors());288 for (const auto & lora : model.loras) {289 res += lora->get_n_nodes();290diff --git a/src/llama-hparams.h b/src/llama-hparams.h291index e2d051edc..812598f69 100644292--- a/src/llama-hparams.h293+++ b/src/llama-hparams.h294@@ -164,6 +164,12 @@ struct llama_hparams {295 float f_embedding_scale = 0.0f;296 float f_attention_scale = 0.0f;297 298+ // HRM-Text recurrence metadata. n_layer remains the expanded KV-cache slot count.299+ uint32_t n_hrm_layer_per_stack = 0;300+ uint32_t n_hrm_h_cycles = 0;301+ uint32_t n_hrm_l_cycles = 0;302+ bool hrm_prefix_lm = false;303+304 // grok-2305 float f_attn_out_scale = 0.0f;306 uint32_t attn_temp_length = 0;307diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp308index 528e4c9c0..8a6e009c6 100644309--- a/src/llama-model-saver.cpp310+++ b/src/llama-model-saver.cpp311@@ -245,6 +245,10 @@ void llama_model_saver::add_kv_from_model() {312 add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count);313 add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);314 // add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???);315+ add_kv(LLM_KV_HRM_LAYERS_PER_STACK, hparams.n_hrm_layer_per_stack);316+ add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles);317+ add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles);318+ add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm);319 320 add_kv(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, true);321 add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, true);322diff --git a/src/llama-model.cpp b/src/llama-model.cpp323index 8bf20a716..a3cc996aa 100644324--- a/src/llama-model.cpp325+++ b/src/llama-model.cpp326@@ -96,6 +96,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params327 return new llama_model_qwen2moe(params);328 case LLM_ARCH_QWEN3:329 return new llama_model_qwen3(params);330+ case LLM_ARCH_HRM_TEXT:331+ return new llama_model_hrm_text(params);332 case LLM_ARCH_QWEN3MOE:333 return new llama_model_qwen3moe(params);334 case LLM_ARCH_QWEN3VL:335@@ -2339,6 +2341,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {336 case LLM_ARCH_PANGU_EMBED:337 case LLM_ARCH_AFMOE:338 case LLM_ARCH_QWEN3NEXT:339+ case LLM_ARCH_HRM_TEXT:340 case LLM_ARCH_MIMO2:341 case LLM_ARCH_STEP35:342 return LLAMA_ROPE_TYPE_NEOX;343diff --git a/src/models/hrm-text.cpp b/src/models/hrm-text.cpp344new file mode 100644345index 000000000..e0a3e9f59346--- /dev/null347+++ b/src/models/hrm-text.cpp348@@ -0,0 +1,183 @@349+#include "models.h"350+351+#include <cmath>352+#include <vector>353+354+void llama_model_hrm_text::load_arch_hparams(llama_model_loader & ml) {355+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);356+ ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale);357+ ml.get_key(LLM_KV_HRM_LAYERS_PER_STACK, hparams.n_hrm_layer_per_stack);358+ ml.get_key(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles);359+ ml.get_key(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles);360+ ml.get_key(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm, false);361+362+ switch (hparams.n_embd) {363+ case 1536: type = LLM_TYPE_1B; break;364+ default: type = LLM_TYPE_UNKNOWN;365+ }366+}367+368+void llama_model_hrm_text::load_arch_tensors(llama_model_loader &) {369+ LLAMA_LOAD_LOCALS;370+371+ const int64_t n_stack = hparams.n_hrm_layer_per_stack;372+ const int64_t n_cycle_slots = n_stack * (hparams.n_hrm_l_cycles + 1);373+374+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);375+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);376+377+ hrm_z_l_init = create_tensor(tn(LLM_TENSOR_HRM_Z_L_INIT), {n_embd}, 0);378+379+ std::vector<bool> loaded_physical(2 * n_stack, false);380+381+ for (int il = 0; il < n_layer; ++il) {382+ auto & layer = layers[il];383+384+ const int64_t layer_in_stack = il % n_stack;385+ const int64_t phase = (il % n_cycle_slots) / n_stack;386+ const bool is_h_stack = phase == int64_t(hparams.n_hrm_l_cycles);387+ const int physical_bid = int((is_h_stack ? n_stack : 0) + layer_in_stack);388+389+ const int flags = loaded_physical[physical_bid] ? TENSOR_DUPLICATED : 0;390+ loaded_physical[physical_bid] = true;391+392+ create_tensor_qkv(layer, physical_bid,393+ n_embd,394+ n_embd_head_k * n_head,395+ n_embd_k_gqa,396+ n_embd_v_gqa,397+ flags);398+399+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", physical_bid), {n_embd, n_embd_head_k * n_head}, flags);400+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", physical_bid), {n_embd_head_k * n_head, n_embd}, flags);401+402+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", physical_bid), {n_embd, n_ff}, flags);403+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", physical_bid), {n_ff, n_embd}, flags);404+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", physical_bid), {n_embd, n_ff}, flags);405+ }406+}407+408+std::unique_ptr<llm_graph_context> llama_model_hrm_text::build_arch_graph(const llm_graph_params & params) const {409+ return std::make_unique<graph>(*this, params);410+}411+412+llama_model_hrm_text::graph::graph(const llama_model & model_, const llm_graph_params & params) : llm_graph_context(params) {413+ const auto & model = static_cast<const llama_model_hrm_text &>(model_);414+415+ GGML_ASSERT(model.tok_embd != nullptr);416+ GGML_ASSERT(model.output != nullptr);417+ GGML_ASSERT(model.hrm_z_l_init != nullptr);418+419+ const int64_t n_embd_head = hparams.n_embd_head_v();420+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());421+ GGML_ASSERT(n_embd_head == n_rot);422+423+ const int64_t n_stack = hparams.n_hrm_layer_per_stack;424+ const int64_t h_cycles = hparams.n_hrm_h_cycles;425+ const int64_t l_cycles = hparams.n_hrm_l_cycles;426+427+ ggml_tensor * inp_pos = build_inp_pos();428+ auto * inp_attn = build_attn_inp_kv();429+ ggml_tensor * inp_out_ids = build_inp_out_ids();430+431+ ggml_tensor * hidden_high = build_inp_embd(model.tok_embd);432+ ggml_tensor * hidden_low = ggml_repeat(ctx0, model.hrm_z_l_init, hidden_high);433+ cb(hidden_low, "hrm_z_l_init", -1);434+435+ const float kq_scale = 1.0f / std::sqrt(float(n_embd_head));436+437+ auto build_stack = [&](ggml_tensor * stack_inp, int slot_offset) -> ggml_tensor * {438+ ggml_tensor * stack_cur = stack_inp;439+440+ for (int layer_idx = 0; layer_idx < n_stack; ++layer_idx) {441+ const int il = slot_offset + layer_idx;442+ const auto & layer = model.layers[il];443+444+ ggml_tensor * inpSA = stack_cur;445+ ggml_tensor * cur = build_norm(stack_cur, nullptr, nullptr, LLM_NORM_RMS, il);446+ cb(cur, "attn_norm", il);447+448+ {449+ ggml_tensor * attn_inp = cur;450+ auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);451+452+ ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, attn_inp, layer.wqkv_gate_s);453+ cb(gate, "attn_gate_proj", il);454+455+ Qcur = ggml_rope_ext(456+ ctx0, Qcur, inp_pos, nullptr,457+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,458+ ext_factor, attn_factor, beta_fast, beta_slow);459+ cb(Qcur, "Qcur_rope", il);460+461+ Kcur = ggml_rope_ext(462+ ctx0, Kcur, inp_pos, nullptr,463+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,464+ ext_factor, attn_factor, beta_fast, beta_slow);465+ cb(Kcur, "Kcur_rope", il);466+467+ cur = build_attn(inp_attn,468+ nullptr, nullptr, nullptr,469+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);470+ cb(cur, "attn_out", il);471+472+ gate = ggml_sigmoid(ctx0, gate);473+ cb(gate, "attn_gate_sig", il);474+475+ cur = ggml_mul(ctx0, cur, gate);476+ cb(cur, "attn_gated", il);477+478+ cur = build_lora_mm(layer.wo, cur, layer.wo_s);479+ cb(cur, "attn_o_proj", il);480+ }481+482+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);483+ cb(ffn_inp, "ffn_inp", il);484+485+ cur = build_norm(ffn_inp, nullptr, nullptr, LLM_NORM_RMS, il);486+ cb(cur, "ffn_norm", il);487+488+ cur = build_ffn(cur,489+ layer.ffn_up, nullptr, layer.ffn_up_s,490+ layer.ffn_gate, nullptr, layer.ffn_gate_s,491+ layer.ffn_down, nullptr, layer.ffn_down_s,492+ nullptr,493+ LLM_FFN_SILU, LLM_FFN_PAR, il);494+ cb(cur, "ffn_out", il);495+496+ cur = ggml_add(ctx0, cur, ffn_inp);497+ cur = build_cvec(cur, il);498+ cb(cur, "hrm_layer_out", il);499+500+ stack_cur = cur;501+ }502+503+ stack_cur = build_norm(stack_cur, nullptr, nullptr, LLM_NORM_RMS, slot_offset);504+ cb(stack_cur, "stack_final_norm", slot_offset);505+ return stack_cur;506+ };507+508+ for (int h = 0; h < h_cycles; ++h) {509+ for (int l = 0; l < l_cycles; ++l) {510+ const int slot_offset = int((h * (l_cycles + 1) + l) * n_stack);511+ hidden_low = build_stack(ggml_add(ctx0, hidden_low, hidden_high), slot_offset);512+ }513+514+ const int slot_offset = int((h * (l_cycles + 1) + l_cycles) * n_stack);515+ hidden_high = build_stack(ggml_add(ctx0, hidden_high, hidden_low), slot_offset);516+ }517+518+ ggml_tensor * cur = hidden_high;519+520+ if (inp_out_ids) {521+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);522+ }523+524+ res->t_embd = cur;525+526+ cur = build_lora_mm(model.output, cur, model.output_s);527+ cb(cur, "result_output", -1);528+529+ res->t_logits = cur;530+ ggml_build_forward_expand(gf, cur);531+}532diff --git a/src/models/models.h b/src/models/models.h533index 7e551eb96..7da6b7f7f 100644534--- a/src/models/models.h535+++ b/src/models/models.h536@@ -515,6 +515,20 @@ struct llama_model_qwen3 : public llama_model_base {537 std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;538 };539 540+struct llama_model_hrm_text : public llama_model_base {541+ llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {}542+ void load_arch_hparams(llama_model_loader & ml) override;543+ void load_arch_tensors(llama_model_loader & ml) override;544+545+ ggml_tensor * hrm_z_l_init = nullptr;546+547+ struct graph : public llm_graph_context {548+ graph(const llama_model & model, const llm_graph_params & params);549+ };550+551+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;552+};553+554 555 struct llama_model_qwen3moe : public llama_model_base {556 llama_model_qwen3moe(const struct llama_model_params & params) : llama_model_base(params) {}557