bcoding/deepseek-llvm-sft-merged-8b
0158
1{2 "architectures": [3 "Qwen3ForCausalLM"4 ],5 "attention_bias": false,6 "attention_dropout": 0.0,7 "bos_token_id": 151643,8 "torch_dtype": "bfloat16",9 "eos_token_id": 151645,10 "head_dim": 128,11 "hidden_act": "silu",12 "hidden_size": 4096,13 "initializer_range": 0.02,14 "intermediate_size": 12288,15 "layer_types": [16 "full_attention",17 "full_attention",18 "full_attention",19 "full_attention",20 "full_attention",21 "full_attention",22 "full_attention",23 "full_attention",24 "full_attention",25 "full_attention",26 "full_attention",27 "full_attention",28 "full_attention",29 "full_attention",30 "full_attention",31 "full_attention",32 "full_attention",33 "full_attention",34 "full_attention",35 "full_attention",36 "full_attention",37 "full_attention",38 "full_attention",39 "full_attention",40 "full_attention",41 "full_attention",42 "full_attention",43 "full_attention",44 "full_attention",45 "full_attention",46 "full_attention",47 "full_attention",48 "full_attention",49 "full_attention",50 "full_attention",51 "full_attention"52 ],53 "max_position_embeddings": 131072,54 "max_window_layers": 36,55 "model_type": "qwen3",56 "num_attention_heads": 32,57 "num_hidden_layers": 36,58 "num_key_value_heads": 8,59 "pad_token_id": 151654,60 "rms_norm_eps": 1e-06,61 "rope_scaling": {62 "attn_factor": 0.8782488562869419,63 "factor": 4.0,64 "original_max_position_embeddings": 32768,65 "rope_type": "yarn"66 },67 "rope_theta": 1000000,68 "sliding_window": null,69 "tie_word_embeddings": false,70 "unsloth_fixed": true,71 "unsloth_version": "2026.8.7",72 "use_cache": false,73 "use_sliding_window": false,74 "vocab_size": 15193675}