hf-tiny-v2/tiny-random-Qwen2MoeForSequenceClassification

1{2  "architectures": [3    "Qwen2MoeForSequenceClassification"4  ],5  "attention_dropout": 0.0,6  "bos_token_id": 1,7  "decoder_sparse_step": 1,8  "dtype": "float32",9  "eos_token_id": 151643,10  "hidden_act": "gelu",11  "hidden_size": 32,12  "initializer_range": 0.02,13  "intermediate_size": 32,14  "layer_types": [15    "full_attention",16    "full_attention"17  ],18  "max_position_embeddings": 512,19  "max_window_layers": 28,20  "mlp_only_layers": [],21  "model_type": "qwen2_moe",22  "moe_intermediate_size": 16,23  "norm_topk_prob": false,24  "num_attention_heads": 2,25  "num_experts": 8,26  "num_experts_per_tok": 2,27  "num_hidden_layers": 2,28  "num_key_value_heads": 2,29  "output_router_logits": false,30  "pad_token_id": 151643,31  "qkv_bias": true,32  "rms_norm_eps": 1e-06,33  "rope_parameters": {34    "rope_theta": 10000.0,35    "rope_type": "default"36  },37  "router_aux_loss_coef": 0.001,38  "shared_expert_intermediate_size": 36,39  "sliding_window": 0,40  "tie_word_embeddings": false,41  "transformers_version": "5.16.0.dev0",42  "use_cache": true,43  "use_sliding_window": false,44  "vocab_size": 15164645}46