tiny-random/hy-v3

1{2  "architectures": [3    "HYV3ForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "bos_token_id": 120000,8  "dtype": "bfloat16",9  "enable_attention_fp32_softmax": false,10  "enable_lm_head_fp32": true,11  "enable_moe_fp32_combine": false,12  "eod_token_id": 120026,13  "eos_token_id": 120025,14  "expert_hidden_dim": 32,15  "first_k_dense_replace": 1,16  "head_dim": 32,17  "hidden_act": "silu",18  "hidden_size": 8,19  "initializer_range": 0.006,20  "intermediate_size": 32,21  "max_position_embeddings": 262144,22  "mlp_bias": false,23  "mlp_layer_types": [24    "dense",25    "sparse",26    "sparse",27    "sparse"28  ],29  "model_type": "hy_v3",30  "moe_intermediate_size": 32,31  "moe_router_enable_expert_bias": true,32  "moe_router_use_sigmoid": true,33  "num_attention_heads": 8,34  "num_experts": 192,35  "num_experts_per_tok": 8,36  "num_hidden_layers": 4,37  "num_key_value_heads": 4,38  "num_nextn_predict_layers": 1,39  "num_shared_experts": 1,40  "output_router_logits": true,41  "pad_token_id": 120002,42  "qk_norm": true,43  "rms_norm_eps": 1e-05,44  "rope_parameters": {45    "rope_theta": 11158840.0,46    "rope_type": "default"47  },48  "route_norm": true,49  "router_scaling_factor": 2.826,50  "sep_token_id": 120007,51  "tie_word_embeddings": false,52  "transformers_version": "5.13.0",53  "use_cache": true,54  "use_grouped_mm": false,55  "vocab_size": 12083256}57