inclusionAI/LLaDA2.2-mini

1{2    "architectures": [3        "LLaDA2MoeModelLM"4    ],5    "attention_dropout": 0.0,6    "auto_map": {7        "AutoConfig": "configuration_llada2_moe.LLaDA2MoeConfig",8        "AutoModel": "modeling_llada2_moe.LLaDA2MoeModel",9        "AutoModelForCausalLM": "modeling_llada2_moe.LLaDA2MoeModelLM"10    },11    "num_hidden_layers": 20,12    "hidden_size": 2048,13    "intermediate_size": 5120,14    "first_k_dense_replace": 1,15    "hidden_act": "silu",16    "max_position_embeddings": 131072,17    "model_type": "llada2_moe",18    "moe_intermediate_size": 512,19    "norm_topk_prob": true,20    "num_experts_per_tok": 8,21    "expert_capacity": 48,22    "block_size": 32,23    "delete_token_id": 156930,24    "split_token_id": 156931,25    "norm_head": false,26    "num_attention_heads": 16,27    "num_experts": 256,28    "num_key_value_heads": 4,29    "rope_theta": 3000000,30    "rope_scaling": null,31    "tie_word_embeddings": false,32    "torch_dtype": "bfloat16",33    "transformers_version": "5.2.0",34    "use_bias": false,35    "use_rmsnorm": true,36    "rms_norm_eps": 1e-06,37    "head_dim": 128,38    "num_shared_experts": 1,39    "use_cache": false,40    "use_qk_norm": true,41    "use_qkv_bias": false,42    "embedding_dropout": 0.0,43    "norm_softmax": false,44    "output_dropout": 0.0,45    "vocab_size": 157184,46    "rotary_dim": 64,47    "using_split_qkv_in_self_attention": false,48    "router_dtype": "fp32",49    "moe_router_enable_expert_bias": true,50    "routed_scaling_factor": 2.5,51    "n_group": 8,52    "topk_group": 4,53    "score_function": "sigmoid",54    "initializer_range": 0.02,55    "max_window_layers": 28,56    "output_router_logits": false,57    "pad_token_id": 156892,58    "partial_rotary_factor": 0.5,59    "use_sliding_window": false60}61