Open4bits/LFM2.5-1.2B-Base-Quantized

1{2  "architectures": [3    "Lfm2ForCausalLM"4  ],5  "block_auto_adjust_ff_dim": true,6  "block_dim": 2048,7  "block_ff_dim": 12288,8  "block_ffn_dim_multiplier": 1.0,9  "block_mlp_init_scale": 1.0,10  "block_multiple_of": 256,11  "block_norm_eps": 1e-05,12  "block_out_init_scale": 1.0,13  "block_use_swiglu": true,14  "block_use_xavier_init": true,15  "bos_token_id": 1,16  "conv_L_cache": 3,17  "conv_bias": false,18  "conv_dim": 2048,19  "conv_use_xavier_init": true,20  "dtype": "float16",21  "eos_token_id": 7,22  "hidden_size": 2048,23  "initializer_range": 0.02,24  "intermediate_size": 12288,25  "layer_types": [26    "conv",27    "conv",28    "full_attention",29    "conv",30    "conv",31    "full_attention",32    "conv",33    "conv",34    "full_attention",35    "conv",36    "full_attention",37    "conv",38    "full_attention",39    "conv",40    "full_attention",41    "conv"42  ],43  "max_position_embeddings": 128000,44  "model_type": "lfm2",45  "norm_eps": 1e-05,46  "num_attention_heads": 32,47  "num_heads": 32,48  "num_hidden_layers": 16,49  "num_key_value_heads": 8,50  "pad_token_id": 0,51  "quantization_config": {52    "_load_in_4bit": true,53    "_load_in_8bit": false,54    "bnb_4bit_compute_dtype": "float16",55    "bnb_4bit_quant_storage": "uint8",56    "bnb_4bit_quant_type": "nf4",57    "bnb_4bit_use_double_quant": true,58    "llm_int8_enable_fp32_cpu_offload": false,59    "llm_int8_has_fp16_weight": false,60    "llm_int8_skip_modules": null,61    "llm_int8_threshold": 6.0,62    "load_in_4bit": true,63    "load_in_8bit": false,64    "quant_method": "bitsandbytes"65  },66  "rope_theta": 1000000.0,67  "tie_embedding": true,68  "transformers_version": "4.57.3",69  "use_cache": true,70  "use_pos_enc": true,71  "vocab_size": 6553672}73