inclusionAI/Ling-lite

1{2    "architectures": [3        "BailingMoeForCausalLM"4    ],5    "attention_dropout": 0.0,6    "auto_map": {7        "AutoConfig": "configuration_bailing_moe.BailingMoeConfig",8        "AutoModel": "modeling_bailing_moe.BailingMoeModel",9        "AutoModelForCausalLM": "modeling_bailing_moe.BailingMoeForCausalLM",10	"AutoModelForTokenClassification": "modeling_bailing_moe.BailingMoeForTokenClassification"11    },12    "eos_token_id": 126081,13    "pad_token_id": 126081,14    "first_k_dense_replace": 0,15    "hidden_act": "silu",16    "hidden_size": 2048,17    "initializer_range": 0.006,18    "intermediate_size": 5632,19    "max_position_embeddings": 32768,20    "model_type": "bailing_moe",21    "moe_intermediate_size": 1408,22    "num_experts": 64,23    "num_shared_experts": 2,24    "norm_topk_prob": true,25    "num_attention_heads": 16,26    "num_experts_per_tok": 6,27    "num_hidden_layers": 28,28    "num_key_value_heads": 4,29    "pretraining_tp": 1,30    "rms_norm_eps": 1e-06,31    "rope_scaling": null,32    "rope_theta": 600000,33    "tie_word_embeddings": false,34    "torch_dtype": "bfloat16",35    "transformers_version": "4.36.0",36    "use_cache": true,37    "use_bias": false,38    "use_qkv_bias": false,39    "vocab_size": 126464,40    "embedding_dropout": 0.0,41    "norm_head": true,42    "norm_softmax": false,43    "output_dropout": 0.0,44    "output_router_logits": false45}46