Ares-Realm-Studios/Trinity-Mini-Base

1{2  "architectures": [3    "AfmoeForCausalLM"4  ],5  "attention_dropout": 0.0,6  "dtype": "bfloat16",7  "global_attn_every_n_layers": 4,8  "head_dim": 128,9  "hidden_act": "silu",10  "hidden_size": 2048,11  "initializer_range": 0.02,12  "intermediate_size": 6144,13  "layer_types": [14    "sliding_attention",15    "sliding_attention",16    "sliding_attention",17    "full_attention",18    "sliding_attention",19    "sliding_attention",20    "sliding_attention",21    "full_attention",22    "sliding_attention",23    "sliding_attention",24    "sliding_attention",25    "full_attention",26    "sliding_attention",27    "sliding_attention",28    "sliding_attention",29    "full_attention",30    "sliding_attention",31    "sliding_attention",32    "sliding_attention",33    "full_attention",34    "sliding_attention",35    "sliding_attention",36    "sliding_attention",37    "full_attention",38    "sliding_attention",39    "sliding_attention",40    "sliding_attention",41    "full_attention",42    "sliding_attention",43    "sliding_attention",44    "sliding_attention",45    "full_attention"46  ],47  "load_balance_coeff": 0.001,48  "max_position_embeddings": 131072,49  "model_type": "afmoe",50  "moe_intermediate_size": 1024,51  "mup_enabled": true,52  "n_group": 1,53  "num_attention_heads": 32,54  "num_dense_layers": 2,55  "num_expert_groups": 1,56  "num_experts": 128,57  "num_experts_per_tok": 8,58  "num_hidden_layers": 32,59  "num_key_value_heads": 4,60  "num_limited_groups": 1,61  "num_shared_experts": 1,62  "rms_norm_eps": 1e-05,63  "rope_scaling": null,64  "rope_theta": 10000,65  "route_norm": true,66  "route_scale": 2.826,67  "score_func": "sigmoid",68  "sliding_window": 2048,69  "tie_word_embeddings": false,70  "topk_group": 1,71  "transformers_version": "4.57.0.dev0",72  "use_cache": true,73  "use_grouped_mm": true,74  "vocab_size": 200192,75  "auto_map": {76    "AutoConfig": "configuration_afmoe.AfmoeConfig",77    "AutoModelForCausalLM": "modeling_afmoe.AfmoeForCausalLM",78    "AutoModel": "modeling_afmoe.AfmoeModel"79  }80}