Azrail/smallm_70_rope

1{2  "architectures": [3    "SmalLmForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.1,7  "auto_map": {8    "AutoConfig": "config.SmalLmConfig",9    "AutoModelForCausalLM": "model.SmalLmForCausalLM"10  },11  "balancing_coef": 0.0001,12  "bos_token_id": 1,13  "embedding_dropout": 0.0,14  "eos_token_id": 0,15  "expert_size": 576,16  "gate_noise": false,17  "head_size": 64,18  "hidden_size": 512,19  "high_rotations": 32,20  "initializer_range": 0.02,21  "intermediate_size": 1024,22  "layer_dropout": 0.1,23  "low_rotations": 1,24  "max_seq_len": 1024,25  "mlp_bias": false,26  "model_type": "smallm",27  "moe_bias": false,28  "moe_period": 2,29  "moe_type": "default",30  "no_moe_layers": 10,31  "noisy_experts": false,32  "num_attention_heads": 8,33  "num_hidden_layers": 20,34  "num_kv_heads": 2,35  "original_seq_len": 1024,36  "pad_token_id": 0,37  "positional_bias_type": "rope",38  "rms_affine": false,39  "rms_norm_eps": 1e-06,40  "rope_base": 100000,41  "routed_experts": 8,42  "shared_experts": 1,43  "sliding_window_attention": true,44  "sliding_window_context": 1024,45  "sliding_window_period": 4,46  "static_residual": true,47  "token_experts": 3,48  "torch_dtype": "float32",49  "transformers_version": "4.50.3",50  "use_cache": true,51  "use_moe": false,52  "vocab_size": 6000053}54