CollectionStudio/Trinity-Mini

1{2  "architectures": [3    "AfmoeForCausalLM"4  ],5  "attention_dropout": 0.0,6  "auto_map": {7    "AutoConfig": "configuration_afmoe.AfmoeConfig",8    "AutoModel": "modeling_afmoe.AfmoeModel",9    "AutoModelForCausalLM": "modeling_afmoe.AfmoeForCausalLM"10  },11  "dtype": "bfloat16",12  "global_attn_every_n_layers": 4,13  "head_dim": 128,14  "hidden_act": "silu",15  "hidden_size": 2048,16  "initializer_range": 0.02,17  "intermediate_size": 6144,18  "layer_types": [19    "sliding_attention",20    "sliding_attention",21    "sliding_attention",22    "full_attention",23    "sliding_attention",24    "sliding_attention",25    "sliding_attention",26    "full_attention",27    "sliding_attention",28    "sliding_attention",29    "sliding_attention",30    "full_attention",31    "sliding_attention",32    "sliding_attention",33    "sliding_attention",34    "full_attention",35    "sliding_attention",36    "sliding_attention",37    "sliding_attention",38    "full_attention",39    "sliding_attention",40    "sliding_attention",41    "sliding_attention",42    "full_attention",43    "sliding_attention",44    "sliding_attention",45    "sliding_attention",46    "full_attention",47    "sliding_attention",48    "sliding_attention",49    "sliding_attention",50    "full_attention"51  ],52  "load_balance_coeff": 0.001,53  "max_position_embeddings": 131072,54  "model_type": "afmoe",55  "moe_intermediate_size": 1024,56  "mup_enabled": true,57  "n_group": 1,58  "num_attention_heads": 32,59  "num_dense_layers": 2,60  "num_expert_groups": 1,61  "num_experts": 128,62  "num_experts_per_tok": 8,63  "num_hidden_layers": 32,64  "num_key_value_heads": 4,65  "num_limited_groups": 1,66  "num_shared_experts": 1,67  "rms_norm_eps": 1e-05,68  "rope_scaling": null,69  "rope_theta": 10000,70  "route_norm": true,71  "route_scale": 2.826,72  "score_func": "sigmoid",73  "sliding_window": 2048,74  "tie_word_embeddings": false,75  "topk_group": 1,76  "transformers_version": "4.57.3",77  "use_cache": true,78  "use_grouped_mm": true,79  "vocab_size": 20019280}81