BensonW/EAI-Final-draft-model

1{2  "architectures": [3    "LlamaForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "bos_token_id": 128000,8  "eos_token_id": [9    128001,10    128008,11    12800912  ],13  "head_dim": 64,14  "hidden_act": "silu",15  "hidden_size": 2048,16  "initializer_range": 0.02,17  "intermediate_size": 8192,18  "max_position_embeddings": 131072,19  "mlp_bias": false,20  "model_type": "llama",21  "num_attention_heads": 32,22  "num_hidden_layers": 16,23  "num_key_value_heads": 8,24  "pretraining_tp": 1,25  "quantization_config": {26    "bits": 4,27    "group_size": 128,28    "modules_to_not_convert": null,29    "quant_method": "awq",30    "version": "gemm",31    "zero_point": true32  },33  "rms_norm_eps": 1e-05,34  "rope_scaling": {35    "factor": 32.0,36    "high_freq_factor": 4.0,37    "low_freq_factor": 1.0,38    "original_max_position_embeddings": 8192,39    "rope_type": "llama3"40  },41  "rope_theta": 500000.0,42  "tie_word_embeddings": true,43  "torch_dtype": "float16",44  "transformers_version": "4.51.3",45  "use_cache": false,46  "vocab_size": 12825647}48