Azrail/smallm_70

1{2  "architectures": [3    "SmalLmForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.1,7  "auto_map": {8    "AutoConfig": "config.SmalLmConfig",9    "AutoModelForCausalLM": "model.SmalLmForCausalLM"10  },11  "balancing_coef": 0.0001,12  "bos_token_id": 1,13  "embedding_dropout": 0.0,14  "eos_token_id": 0,15  "expert_size": 576,16  "gate_noise": false,17  "head_size": 64,18  "hidden_size": 512,19  "high_rotations": 32,20  "initializer_range": 0.02,21  "intermediate_size": 1024,22  "layer_dropout": 0.1,23  "low_rotations": 1,24  "max_seq_len": 1024,25  "mlp_bias": false,26  "model_type": "smallm",27  "moe_bias": false,28  "moe_period": 2,29  "no_moe_layers": 10,30  "noisy_experts": false,31  "num_attention_heads": 8,32  "num_hidden_layers": 20,33  "num_kv_heads": 2,34  "original_seq_len": 1024,35  "pad_token_id": 0,36  "positional_bias_type": "rope",37  "rms_affine": false,38  "rms_norm_eps": 1e-06,39  "rope_base": 100000,40  "routed_experts": 8,41  "shared_experts": 1,42  "sliding_window_attention": true,43  "sliding_window_context": 1024,44  "sliding_window_period": 4,45  "static_residual": true,46  "token_experts": 3,47  "torch_dtype": "float32",48  "transformers_version": "4.50.3",49  "use_cache": true,50  "use_moe": false,51  "vocab_size": 6000052}53