dotan1111/BetaReconstruct_Configuration3

1{2  "_name_or_path": "/groups/pupko/edodotan/BetaAlignPart3/test_architecture/mix_training_state_space_models_10_15_0p200_different_seed_v2/ZAMBA/layers_12_d_1408_e-4_ZAMBA/checkpoint-20480/",3  "architectures": [4    "ZambaForCausalLM"5  ],6  "attention_dropout": 0.0,7  "attention_head_dim": 176,8  "attention_hidden_size": 2816,9  "attn_layer_offset": 4,10  "attn_layer_period": 6,11  "bos_token_id": 1,12  "eos_token_id": 4,13  "hidden_act": "gelu",14  "hidden_mamba_act": "silu",15  "hidden_size": 1408,16  "initializer_range": 0.02,17  "intermediate_size": 4928,18  "layers_block_type": [19    "mamba",20    "mamba",21    "hybrid",22    "mamba",23    "mamba",24    "mamba",25    "mamba",26    "hybrid",27    "mamba",28    "mamba",29    "mamba",30    "mamba"31  ],32  "mamba_conv_bias": true,33  "mamba_d_conv": 4,34  "mamba_d_state": 16,35  "mamba_dt_rank": 88,36  "mamba_expand": 2,37  "mamba_proj_bias": false,38  "max_position_embeddings": 2048,39  "model_type": "zamba",40  "n_mamba_heads": 2,41  "num_attention_heads": 16,42  "num_heads": 44,43  "num_hidden_layers": 12,44  "num_key_value_heads": 16,45  "num_logits_to_keep": 1,46  "pad_token_id": 3,47  "rms_norm_eps": 1e-05,48  "time_step_floor": 0.0001,49  "time_step_max": 0.1,50  "time_step_min": 0.001,51  "torch_dtype": "float32",52  "transformers_version": "4.48.1",53  "use_cache": true,54  "use_mamba_kernels": true,55  "vocab_size": 640056}57