algorithms-learning/correctional-GPTaM

1{2  "architectures": [3    "Lfm2ForCausalLM"4  ],5  "block_auto_adjust_ff_dim": false,6  "block_dim": 2048,7  "block_ffn_dim_multiplier": 1.0,8  "block_mlp_init_scale": 1.0,9  "block_multiple_of": 256,10  "block_norm_eps": 1e-05,11  "block_out_init_scale": 1.0,12  "block_use_swiglu": true,13  "block_use_xavier_init": true,14  "bos_token_id": 124894,15  "conv_L_cache": 3,16  "conv_bias": false,17  "conv_dim": 2048,18  "conv_use_xavier_init": true,19  "dtype": "float16",20  "eos_token_id": 124900,21  "full_attn_idxs": null,22  "hidden_size": 2048,23  "initializer_range": 0.02,24  "intermediate_size": 10752,25  "layer_types": [26    "conv",27    "conv",28    "full_attention",29    "conv",30    "conv",31    "full_attention",32    "conv",33    "full_attention",34    "conv",35    "full_attention",36    "conv",37    "full_attention",38    "conv",39    "conv",40    "conv",41    "full_attention",42    "conv",43    "conv",44    "full_attention",45    "conv",46    "conv",47    "full_attention",48    "conv",49    "conv"50  ],51  "max_position_embeddings": 128000,52  "model_type": "lfm2",53  "norm_eps": 1e-05,54  "num_attention_heads": 32,55  "num_heads": 32,56  "num_hidden_layers": 24,57  "num_key_value_heads": 8,58  "pad_token_id": 124893,59  "rope_parameters": {60    "rope_theta": 10000000.0,61    "rope_type": "default"62  },63  "tie_word_embeddings": true,64  "transformers_version": "5.15.0",65  "use_cache": true,66  "use_pos_enc": true,67  "vocab_size": 12800068}69