Shekswess/tiny-think-sft-math-stem-loss-dft-bf16-e2-bs8

1{2  "architectures": [3    "Llama4ForCausalLM"4  ],5  "attention_bias": false,6  "attention_chunk_size": 32768,7  "attention_dropout": 0.0,8  "attn_scale": 0.1,9  "attn_temperature_tuning": false,10  "bos_token_id": 128000,11  "dtype": "bfloat16",12  "eos_token_id": 128009,13  "floor_scale": 8192,14  "for_llm_compressor": false,15  "head_dim": 64,16  "hidden_act": "silu",17  "hidden_size": 576,18  "initializer_range": 0.02,19  "interleave_moe_layer_step": 0,20  "intermediate_size": 8192,21  "intermediate_size_mlp": 2048,22  "layer_types": [23    "full_attention",24    "full_attention",25    "full_attention",26    "full_attention",27    "full_attention",28    "full_attention",29    "full_attention",30    "full_attention",31    "full_attention",32    "full_attention",33    "full_attention",34    "full_attention",35    "full_attention",36    "full_attention",37    "full_attention"38  ],39  "max_position_embeddings": 4096,40  "model_type": "llama4_text",41  "moe_layers": [],42  "no_rope_layers": [43    1,44    1,45    1,46    1,47    1,48    1,49    1,50    1,51    1,52    1,53    1,54    1,55    1,56    1,57    158  ],59  "num_attention_heads": 9,60  "num_experts_per_tok": 0,61  "num_hidden_layers": 15,62  "num_key_value_heads": 3,63  "num_local_experts": 0,64  "output_router_logits": false,65  "pad_token_id": 128009,66  "rms_norm_eps": 1e-05,67  "rope_scaling": null,68  "rope_theta": 1000000.0,69  "router_aux_loss_coef": 0.001,70  "router_jitter_noise": 0.0,71  "tie_word_embeddings": true,72  "transformers_version": "4.57.3",73  "use_cache": false,74  "use_qk_norm": true,75  "vocab_size": 12825676}77