ServiceNow-AI/Apriel-5B-Base

1{2  "_name_or_path": "ServiceNow-AI/Apriel-5B-Base",3  "architectures": [4    "AprielForCausalLM"5  ],6  "auto_map": {7    "AutoConfig": "configuration_apriel.AprielConfig",8    "AutoModelForCausalLM": "modeling_apriel.AprielForCausalLM",9    "AutoModel": "modeling_apriel.AprielModel"10  },11  "attention_bias": false,12  "attention_dropout": 0.0,13  "bos_token_id": 1,14  "eos_token_id": 2,15  "head_dim": 128,16  "hidden_act": "silu",17  "hidden_size": 4096,18  "initializer_range": 0.02,19  "intermediate_size": 8192,20  "max_position_embeddings": 16384,21  "mlp_bias": false,22  "model_type": "apriel",23  "num_attention_heads": 24,24  "num_hidden_layers": 28,25  "num_key_value_heads": 8,26  "pretraining_tp": 1,27  "rms_norm_eps": 1e-05,28  "rope_scaling": {29    "attention_factor": null,30    "beta_fast": 32.0,31    "beta_slow": 1.0,32    "factor": 32.0,33    "original_max_position_embeddings": 4096,34    "rope_type": "yarn"35  },36  "rope_theta": 1000000.0,37  "tie_word_embeddings": false,38  "torch_dtype": "bfloat16",39  "transformers_version": "4.48.3",40  "use_cache": true,41  "vocab_size": 13107242}43