FrontiersMind/Nandi-Mini-150M-Instruct

1{2  "architectures": [3    "NandiForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "auto_map": {8    "AutoConfig": "configuration_nandi.NandiConfig",9    "AutoModel": "modeling_nandi.NandiModel",10    "AutoModelForCausalLM": "modeling_nandi.NandiForCausalLM"11  },12  "bos_token_id": 1,13  "dtype": "bfloat16",14  "embedding_rank": 196,15  "eos_token_id": 0,16  "factorized_embedding": true,17  "head_dim": 52,18  "hidden_act": "silu",19  "hidden_size": 832,20  "initializer_range": 0.02,21  "intermediate_size": 2496,22  "layer_sharing": true,23  "layer_sharing_repeats": 2,24  "max_position_embeddings": 2048,25  "mlp_bias": false,26  "model_type": "nandi",27  "num_attention_heads": 16,28  "num_hidden_layers": 16,29  "num_key_value_heads": 4,30  "pad_token_id": 0,31  "pretraining_tp": 1,32  "rms_norm_eps": 1e-05,33  "rope_parameters": {34    "rope_theta": 100000,35    "rope_type": "default"36  },37  "tie_word_embeddings": true,38  "transformers_version": "5.4.0",39  "use_cache": false,40  "vocab_size": 13107241}42