CollectionStudio/Trinity-Nano-Base-Pre-Anneal

1{2  "architectures": [3    "AfmoeForCausalLM"4  ],5  "attention_dropout": 0.0,6  "auto_map": {7    "AutoConfig": "configuration_afmoe.AfmoeConfig",8    "AutoModel": "modeling_afmoe.AfmoeModel",9    "AutoModelForCausalLM": "modeling_afmoe.AfmoeForCausalLM"10  },11  "dtype": "bfloat16",12  "global_attn_every_n_layers": 4,13  "head_dim": 128,14  "hidden_act": "silu",15  "hidden_size": 1024,16  "initializer_range": 0.02,17  "intermediate_size": 3072,18  "layer_types": [19    "sliding_attention",20    "sliding_attention",21    "sliding_attention",22    "full_attention",23    "sliding_attention",24    "sliding_attention",25    "sliding_attention",26    "full_attention",27    "sliding_attention",28    "sliding_attention",29    "sliding_attention",30    "full_attention",31    "sliding_attention",32    "sliding_attention",33    "sliding_attention",34    "full_attention",35    "sliding_attention",36    "sliding_attention",37    "sliding_attention",38    "full_attention",39    "sliding_attention",40    "sliding_attention",41    "sliding_attention",42    "full_attention",43    "sliding_attention",44    "sliding_attention",45    "sliding_attention",46    "full_attention",47    "sliding_attention",48    "sliding_attention",49    "sliding_attention",50    "full_attention",51    "sliding_attention",52    "sliding_attention",53    "sliding_attention",54    "full_attention",55    "sliding_attention",56    "sliding_attention",57    "sliding_attention",58    "full_attention",59    "sliding_attention",60    "sliding_attention",61    "sliding_attention",62    "full_attention",63    "sliding_attention",64    "sliding_attention",65    "sliding_attention",66    "full_attention",67    "sliding_attention",68    "sliding_attention",69    "sliding_attention",70    "full_attention",71    "sliding_attention",72    "sliding_attention",73    "sliding_attention",74    "full_attention"75  ],76  "load_balance_coeff": 0.001,77  "max_position_embeddings": 4096,78  "model_type": "afmoe",79  "moe_intermediate_size": 256,80  "mup_enabled": true,81  "n_group": 1,82  "num_attention_heads": 8,83  "num_dense_layers": 2,84  "num_expert_groups": 1,85  "num_experts": 128,86  "num_experts_per_tok": 8,87  "num_hidden_layers": 56,88  "num_key_value_heads": 2,89  "num_limited_groups": 1,90  "num_shared_experts": 1,91  "rms_norm_eps": 1e-05,92  "rope_scaling": null,93  "rope_theta": 10000,94  "route_norm": true,95  "route_scale": 2.826,96  "score_func": "sigmoid",97  "sliding_window": 2048,98  "tie_word_embeddings": false,99  "topk_group": 1,100  "transformers_version": "4.57.3",101  "use_cache": true,102  "use_grouped_mm": true,103  "vocab_size": 200192104}105