jdopensource/JoyAI-LLM-Flash-Base

1{2  "architectures": [3    "DeepseekV3ForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "auto_map": {8    "AutoConfig": "configuration_deepseek.DeepseekV3Config",9    "AutoModel": "modeling_deepseek.DeepseekV3Model",10    "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"11  },12  "bos_token_id": 0,13  "eos_token_id": 1,14  "ep_size": 1,15  "first_k_dense_replace": 1,16  "hidden_act": "silu",17  "hidden_size": 2048,18  "initializer_range": 0.02,19  "intermediate_size": 7168,20  "kv_lora_rank": 512,21  "max_position_embeddings": 131072,22  "model_type": "joyai_llm_flash",23  "moe_intermediate_size": 768,24  "moe_layer_freq": 1,25  "n_group": 1,26  "n_routed_experts": 256,27  "n_shared_experts": 1,28  "norm_topk_prob": true,29  "num_attention_heads": 32,30  "num_experts_per_tok": 8,31  "num_hidden_layers": 40,32  "num_key_value_heads": 32,33  "num_nextn_predict_layers": 1,34  "q_lora_rank": 1536,35  "qk_nope_head_dim": 128,36  "qk_rope_head_dim": 64,37  "rms_norm_eps": 1e-06,38  "rope_theta": 32000000,39  "routed_scaling_factor": 2.5,40  "scoring_func": "sigmoid",41  "tie_word_embeddings": false,42  "topk_group": 1,43  "topk_method": "noaux_tc",44  "torch_dtype": "bfloat16",45  "transformers_version": "4.44.2",46  "use_cache": true,47  "v_head_dim": 128,48  "vocab_size": 12928049}50