pmahdavi/Olmo-3-7B-Think-Math-Code

1{2  "architectures": [3    "Olmo3ForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "dtype": "bfloat16",8  "eos_token_id": 100257,9  "hidden_act": "silu",10  "hidden_size": 4096,11  "initializer_range": 0.02,12  "intermediate_size": 11008,13  "layer_types": [14    "sliding_attention",15    "sliding_attention",16    "sliding_attention",17    "full_attention",18    "sliding_attention",19    "sliding_attention",20    "sliding_attention",21    "full_attention",22    "sliding_attention",23    "sliding_attention",24    "sliding_attention",25    "full_attention",26    "sliding_attention",27    "sliding_attention",28    "sliding_attention",29    "full_attention",30    "sliding_attention",31    "sliding_attention",32    "sliding_attention",33    "full_attention",34    "sliding_attention",35    "sliding_attention",36    "sliding_attention",37    "full_attention",38    "sliding_attention",39    "sliding_attention",40    "sliding_attention",41    "full_attention",42    "sliding_attention",43    "sliding_attention",44    "sliding_attention",45    "full_attention"46  ],47  "max_position_embeddings": 65536,48  "model_type": "olmo3",49  "num_attention_heads": 32,50  "num_hidden_layers": 32,51  "num_key_value_heads": 32,52  "pad_token_id": 100277,53  "rms_norm_eps": 1e-06,54  "rope_scaling": {55    "attention_factor": 1.2079441541679836,56    "beta_fast": 32,57    "beta_slow": 1,58    "factor": 8.0,59    "original_max_position_embeddings": 8192,60    "rope_type": "yarn"61  },62  "rope_theta": 500000,63  "sliding_window": 4096,64  "tie_word_embeddings": false,65  "transformers_version": "4.57.3",66  "use_cache": true,67  "vocab_size": 10027868}69