poolside/Laguna-M.1-base

1{2  "architectures": [3    "LagunaForCausalLM"4  ],5  "auto_map": {6    "AutoConfig": "configuration_laguna.LagunaConfig",7    "AutoModelForCausalLM": "modeling_laguna.LagunaForCausalLM"8  },9  "model_type": "laguna",10  "vocab_size": 100352,11  "hidden_size": 4096,12  "intermediate_size": 16384,13  "num_hidden_layers": 70,14  "num_attention_heads": 64,15  "num_key_value_heads": 8,16  "head_dim": 128,17  "max_position_embeddings": 262144,18  "attention_bias": false,19  "attention_dropout": 0.0,20  "rms_norm_eps": 1e-06,21  "num_experts": 256,22  "num_experts_per_tok": 16,23  "moe_intermediate_size": 1024,24  "shared_expert_intermediate_size": 1024,25  "norm_topk_prob": true,26  "router_aux_loss_coef": 0.00025,27  "decoder_sparse_step": 1,28  "mlp_only_layers": [29    0,30    1,31    232  ],33  "bos_token_id": 2,34  "eos_token_id": [35    2,36    2437  ],38  "pad_token_id": 9,39  "tie_word_embeddings": false,40  "use_cache": true,41  "torch_dtype": "bfloat16",42  "gating": "per-element",43  "sliding_window": 0,44  "rope_parameters": {45    "full_attention": {46      "rope_theta": 500000.0,47      "rope_type": "yarn",48      "factor": 64.0,49      "original_max_position_embeddings": 4096,50      "beta_slow": 1.0,51      "beta_fast": 64.0,52      "attention_factor": 1.4158883083359672,53      "partial_rotary_factor": 1.054    }55  },56  "moe_apply_router_weight_on_input": false,57  "partial_rotary_factor": 1.0,58  "mlp_layer_types": [59    "dense",60    "dense",61    "dense",62    "sparse",63    "sparse",64    "sparse",65    "sparse",66    "sparse",67    "sparse",68    "sparse",69    "sparse",70    "sparse",71    "sparse",72    "sparse",73    "sparse",74    "sparse",75    "sparse",76    "sparse",77    "sparse",78    "sparse",79    "sparse",80    "sparse",81    "sparse",82    "sparse",83    "sparse",84    "sparse",85    "sparse",86    "sparse",87    "sparse",88    "sparse",89    "sparse",90    "sparse",91    "sparse",92    "sparse",93    "sparse",94    "sparse",95    "sparse",96    "sparse",97    "sparse",98    "sparse",99    "sparse",100    "sparse",101    "sparse",102    "sparse",103    "sparse",104    "sparse",105    "sparse",106    "sparse",107    "sparse",108    "sparse",109    "sparse",110    "sparse",111    "sparse",112    "sparse",113    "sparse",114    "sparse",115    "sparse",116    "sparse",117    "sparse",118    "sparse",119    "sparse",120    "sparse",121    "sparse",122    "sparse",123    "sparse",124    "sparse",125    "sparse",126    "sparse",127    "sparse",128    "sparse"129  ],130  "moe_routed_scaling_factor": 1.0131}132