KingRaRa/Lens-Turbo

1{2  "architectures": [3    "GptOssForCausalLM"4  ],5  "attention_bias": true,6  "attention_dropout": 0.0,7  "bos_token_id": null,8  "dtype": "bfloat16",9  "eos_token_id": 200002,10  "experts_per_token": 4,11  "head_dim": 64,12  "hidden_act": "silu",13  "hidden_size": 2880,14  "initial_context_length": 4096,15  "initializer_range": 0.02,16  "intermediate_size": 2880,17  "layer_types": [18    "sliding_attention",19    "full_attention",20    "sliding_attention",21    "full_attention",22    "sliding_attention",23    "full_attention",24    "sliding_attention",25    "full_attention",26    "sliding_attention",27    "full_attention",28    "sliding_attention",29    "full_attention",30    "sliding_attention",31    "full_attention",32    "sliding_attention",33    "full_attention",34    "sliding_attention",35    "full_attention",36    "sliding_attention",37    "full_attention",38    "sliding_attention",39    "full_attention",40    "sliding_attention",41    "full_attention"42  ],43  "max_position_embeddings": 131072,44  "model_type": "gpt_oss",45  "num_attention_heads": 64,46  "num_experts_per_tok": 4,47  "num_hidden_layers": 24,48  "num_key_value_heads": 8,49  "num_local_experts": 32,50  "output_router_logits": false,51  "pad_token_id": 199999,52  "quantization_config": {53    "modules_to_not_convert": [54      "model.layers.*.self_attn",55      "model.layers.*.mlp.router",56      "model.embed_tokens",57      "lm_head"58    ],59    "quant_method": "mxfp4"60  },61  "rms_norm_eps": 1e-05,62  "rope_parameters": {63    "beta_fast": 32.0,64    "beta_slow": 1.0,65    "factor": 32.0,66    "original_max_position_embeddings": 4096,67    "rope_theta": 150000,68    "rope_type": "yarn",69    "truncate": false70  },71  "router_aux_loss_coef": 0.9,72  "sliding_window": 128,73  "swiglu_limit": 7.0,74  "tie_word_embeddings": false,75  "transformers_version": "5.8.0",76  "use_cache": true,77  "vocab_size": 20108878}79