limloop/whiff-mamba2-20M

1{2  "architectures": [3    "Mamba2ForCausalLM"4  ],5  "bos_token_id": 6,6  "chunk_size": 256,7  "conv_kernel": 4,8  "eos_token_id": 6,9  "expand": 1.5,10  "head_dim": 64,11  "hidden_act": "silu",12  "hidden_size": 512,13  "initializer_range": 0.1,14  "layer_norm_epsilon": 1e-05,15  "model_type": "mamba2",16  "n_groups": 2,17  "num_heads": 12,18  "num_hidden_layers": 9,19  "pad_token_id": 7,20  "rescale_prenorm_residual": false,21  "residual_in_fp32": true,22  "rms_norm": true,23  "state_size": 64,24  "tie_word_embeddings": false,25  "time_step_floor": 0.0001,26  "time_step_limit": [27    0.0,28    {29      "__float__": "Infinity"30    }31  ],32  "time_step_max": 0.1,33  "time_step_min": 0.001,34  "time_step_rank": 32,35  "torch_dtype": "float32",36  "transformers_version": "4.54.1",37  "use_bias": false,38  "use_cache": true,39  "use_conv_bias": true,40  "vocab_size": 819241}42