Modelpublic

UniversalComputingResearch/Limen0.2B

sourceHugging Faceapache-2.0updated 2mo agoView on Hugging Face

9likes63downloads

config.json27 linesDownload Raw Back to root

1{2  "architectures": [3    "GPTForCausalLM"4  ],5  "auto_map": {6    "AutoConfig": "config.GPTConfig",7    "AutoModelForCausalLM": "model.GPTForCausalLM"8  },9  "block_size": 1024,10  "dtype": "float32",11  "head_dim": 128,12  "hidden_size": 768,13  "intermediate_size": 1920,14  "labels_are_shifted": true,15  "max_position_embeddings": 1024,16  "model_type": "gpt",17  "num_attention_heads": 6,18  "num_hidden_layers": 35,19  "num_key_value_heads": 2,20  "rms_norm_eps": 1e-06,21  "rope_theta": 100000.0,22  "tie_word_embeddings": true,23  "transformers_version": "5.14.1",24  "vocab_size": 16384,25  "xsa_projection": true26}27