DarkSca/GenZ_BitNet

1{2  "architectures": [3    "BitNetForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "auto_map": {8    "AutoConfig": "configuration_bitnet.BitNetConfig",9    "AutoModelForCausalLM": "modeling_bitnet.BitNetForCausalLM"10  },11  "bos_token_id": 128000,12  "dtype": "float32",13  "eos_token_id": 128001,14  "hidden_act": "relu2",15  "hidden_size": 2560,16  "initializer_range": 0.02,17  "intermediate_size": 6912,18  "max_position_embeddings": 4096,19  "model_type": "bitnet",20  "num_attention_heads": 20,21  "num_hidden_layers": 30,22  "num_key_value_heads": 5,23  "quantization_config": {24    "linear_class": "autobitlinear",25    "modules_to_not_convert": null,26    "quant_method": "bitnet",27    "quantization_mode": "online",28    "rms_norm_eps": 1e-06,29    "use_rms_norm": false30  },31  "rms_norm_eps": 1e-05,32  "rope_theta": 500000.0,33  "tie_word_embeddings": true,34  "transformers_version": "4.57.1",35  "use_cache": true,36  "vocab_size": 12825637}38