okestro-ai-lab/FastSLM

1{2  "architectures": [3    "FastSLMForConditionalGeneration"4  ],5  "encoder_config": {6    "compression_size": 50,7    "model_type": "fastslm_speech_encoder",8    "n_ctx": 1500,9    "n_head": 20,10    "n_layer": 32,11    "n_mels": 128,12    "n_state": 1280,13    "stage_tokens": [14      80,15      80,16      8017    ]18  },19  "llm_config": {20    "_name_or_path": "Qwen/Qwen3-4B",21    "architectures": [22      "Qwen3ForCausalLM"23    ],24    "attention_bias": false,25    "attention_dropout": 0.0,26    "bos_token_id": 151643,27    "eos_token_id": 151645,28    "head_dim": 128,29    "hidden_act": "silu",30    "hidden_size": 2560,31    "initializer_range": 0.02,32    "intermediate_size": 9728,33    "layer_types": [34      "full_attention",35      "full_attention",36      "full_attention",37      "full_attention",38      "full_attention",39      "full_attention",40      "full_attention",41      "full_attention",42      "full_attention",43      "full_attention",44      "full_attention",45      "full_attention",46      "full_attention",47      "full_attention",48      "full_attention",49      "full_attention",50      "full_attention",51      "full_attention",52      "full_attention",53      "full_attention",54      "full_attention",55      "full_attention",56      "full_attention",57      "full_attention",58      "full_attention",59      "full_attention",60      "full_attention",61      "full_attention",62      "full_attention",63      "full_attention",64      "full_attention",65      "full_attention",66      "full_attention",67      "full_attention",68      "full_attention",69      "full_attention"70    ],71    "max_position_embeddings": 40960,72    "max_window_layers": 36,73    "model_type": "qwen3",74    "num_attention_heads": 32,75    "num_hidden_layers": 36,76    "num_key_value_heads": 8,77    "rms_norm_eps": 1e-06,78    "rope_scaling": null,79    "rope_parameters": {80      "rope_type": "default",81      "rope_theta": 100000082    },83    "sliding_window": null,84    "tie_word_embeddings": true,85    "torch_dtype": "bfloat16",86    "use_cache": true,87    "use_sliding_window": false,88    "vocab_size": 15193689  },90  "llm_modules": [91    "q_proj",92    "k_proj",93    "v_proj",94    "o_proj",95    "gate_proj",96    "up_proj",97    "down_proj"98  ],99  "lora_a": 64,100  "lora_r": 16,101  "low_resource": false,102  "model_type": "fastslm",103  "auto_map": {104    "AutoConfig": "configuration_fastslm.FastSLMConfig",105    "AutoModel": "modeling_fastslm.FastSLMForConditionalGeneration",106    "AutoModelForCausalLM": "modeling_fastslm.FastSLMForConditionalGeneration"107  },108  "torch_dtype": "float32",109  "transformers_version": "4.53.1"110}