tcclaviger/Laguna-S-2.1-RFA_L
045
1{2 "architectures": [3 "LagunaForCausalLM"4 ],5 "auto_map": {6 "AutoConfig": "configuration_laguna.LagunaConfig",7 "AutoModelForCausalLM": "modeling_laguna.LagunaForCausalLM"8 },9 "model_type": "laguna",10 "vocab_size": 100352,11 "hidden_size": 3072,12 "intermediate_size": 12288,13 "num_hidden_layers": 48,14 "num_attention_heads": 48,15 "num_key_value_heads": 8,16 "head_dim": 128,17 "max_position_embeddings": 1048576,18 "attention_bias": false,19 "attention_dropout": 0.0,20 "rms_norm_eps": 1e-06,21 "num_experts": 256,22 "num_experts_per_tok": 10,23 "moe_intermediate_size": 1024,24 "shared_expert_intermediate_size": 1024,25 "norm_topk_prob": true,26 "router_aux_loss_coef": 0.0,27 "decoder_sparse_step": 1,28 "mlp_only_layers": [29 030 ],31 "bos_token_id": 2,32 "eos_token_id": [33 2,34 2435 ],36 "pad_token_id": 9,37 "tie_word_embeddings": false,38 "use_cache": true,39 "torch_dtype": "bfloat16",40 "gating": "per-head",41 "sliding_window": 512,42 "rope_parameters": {43 "full_attention": {44 "rope_theta": 500000.0,45 "rope_type": "yarn",46 "factor": 128.0,47 "original_max_position_embeddings": 8192,48 "beta_slow": 1.0,49 "beta_fast": 32.0,50 "attention_factor": 1.4852030263919618,51 "partial_rotary_factor": 0.552 },53 "sliding_attention": {54 "rope_type": "default",55 "rope_theta": 10000.0,56 "partial_rotary_factor": 1.057 }58 },59 "layer_types": [60 "full_attention",61 "sliding_attention",62 "sliding_attention",63 "sliding_attention",64 "full_attention",65 "sliding_attention",66 "sliding_attention",67 "sliding_attention",68 "full_attention",69 "sliding_attention",70 "sliding_attention",71 "sliding_attention",72 "full_attention",73 "sliding_attention",74 "sliding_attention",75 "sliding_attention",76 "full_attention",77 "sliding_attention",78 "sliding_attention",79 "sliding_attention",80 "full_attention",81 "sliding_attention",82 "sliding_attention",83 "sliding_attention",84 "full_attention",85 "sliding_attention",86 "sliding_attention",87 "sliding_attention",88 "full_attention",89 "sliding_attention",90 "sliding_attention",91 "sliding_attention",92 "full_attention",93 "sliding_attention",94 "sliding_attention",95 "sliding_attention",96 "full_attention",97 "sliding_attention",98 "sliding_attention",99 "sliding_attention",100 "full_attention",101 "sliding_attention",102 "sliding_attention",103 "sliding_attention",104 "full_attention",105 "sliding_attention",106 "sliding_attention",107 "sliding_attention"108 ],109 "moe_apply_router_weight_on_input": false,110 "mlp_layer_types": [111 "dense",112 "sparse",113 "sparse",114 "sparse",115 "sparse",116 "sparse",117 "sparse",118 "sparse",119 "sparse",120 "sparse",121 "sparse",122 "sparse",123 "sparse",124 "sparse",125 "sparse",126 "sparse",127 "sparse",128 "sparse",129 "sparse",130 "sparse",131 "sparse",132 "sparse",133 "sparse",134 "sparse",135 "sparse",136 "sparse",137 "sparse",138 "sparse",139 "sparse",140 "sparse",141 "sparse",142 "sparse",143 "sparse",144 "sparse",145 "sparse",146 "sparse",147 "sparse",148 "sparse",149 "sparse",150 "sparse",151 "sparse",152 "sparse",153 "sparse",154 "sparse",155 "sparse",156 "sparse",157 "sparse",158 "sparse"159 ],160 "gating_types": [161 "per_head",162 "per_head",163 "per_head",164 "per_head",165 "per_head",166 "per_head",167 "per_head",168 "per_head",169 "per_head",170 "per_head",171 "per_head",172 "per_head",173 "per_head",174 "per_head",175 "per_head",176 "per_head",177 "per_head",178 "per_head",179 "per_head",180 "per_head",181 "per_head",182 "per_head",183 "per_head",184 "per_head",185 "per_head",186 "per_head",187 "per_head",188 "per_head",189 "per_head",190 "per_head",191 "per_head",192 "per_head",193 "per_head",194 "per_head",195 "per_head",196 "per_head",197 "per_head",198 "per_head",199 "per_head",200 "per_head",201 "per_head",202 "per_head",203 "per_head",204 "per_head",205 "per_head",206 "per_head",207 "per_head",208 "per_head"209 ],210 "moe_routed_scaling_factor": 2.5,211 "num_attention_heads_per_layer": [212 48,213 72,214 72,215 72,216 48,217 72,218 72,219 72,220 48,221 72,222 72,223 72,224 48,225 72,226 72,227 72,228 48,229 72,230 72,231 72,232 48,233 72,234 72,235 72,236 48,237 72,238 72,239 72,240 48,241 72,242 72,243 72,244 48,245 72,246 72,247 72,248 48,249 72,250 72,251 72,252 48,253 72,254 72,255 72,256 48,257 72,258 72,259 72260 ],261 "moe_router_logit_softcapping": 0.0,262 "quantization_config": {263 "config_groups": {264 "group_0": {265 "format": "rfa-pack-quantized",266 "input_activations": null,267 "output_activations": null,268 "targets": [269 "Linear"270 ],271 "weights": {272 "num_bits": 4,273 "type": "float",274 "group_size": 16,275 "grid": "iq4_nl",276 "codebook": [277 -127,278 -104,279 -83,280 -65,281 -49,282 -35,283 -22,284 -10,285 1,286 13,287 25,288 38,289 53,290 69,291 89,292 113293 ],294 "scale_dtype": "torch.int8",295 "scale_exp_dtype": "torch.int8",296 "scale_encoding": "block_float_i8m_i8e",297 "symmetric": false,298 "rotation": "hadamard16"299 }300 }301 },302 "format": "rfa-pack-quantized",303 "quant_method": "rfa",304 "quantization_status": "compressed",305 "version": "rfa-1.0",306 "ignore": [307 "re:.*\\.self_attn\\..*",308 "re:.*\\.mlp\\.gate\\.weight$",309 "re:^lm_head",310 "re:.*\\.mlp\\.shared_expert\\..*",311 "re:.*\\.layers\\.0\\.mlp\\..*"312 ]313 }314}