Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2025 The HuggingFace Team. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""openai model configuration"""16 17from ...configuration_utils import PretrainedConfig, layer_type_validation18from ...modeling_rope_utils import rope_config_validation19 20 21class GptOssConfig(PretrainedConfig):22    r"""23    This will yield a configuration to that of the BERT24    [google-bert/bert-base-uncased](https://huggingface.co/google-bert/bert-base-uncased) architecture.25 26    """27 28    model_type = "gpt_oss"29    base_model_pp_plan = {30        "embed_tokens": (["input_ids"], ["inputs_embeds"]),31        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),32        "norm": (["hidden_states"], ["hidden_states"]),33    }34    base_model_tp_plan = {35        "layers.*.self_attn.q_proj": "colwise",36        "layers.*.self_attn.k_proj": "colwise",37        "layers.*.self_attn.v_proj": "colwise",38        "layers.*.self_attn.o_proj": "rowwise",39        "layers.*.self_attn.sinks": "local_rowwise",40        "layers.*.mlp.experts": "gather",41        "layers.*.mlp.router": "ep_router",42        "layers.*.mlp.experts.gate_up_proj": "grouped_gemm",43        "layers.*.mlp.experts.gate_up_proj_bias": "grouped_gemm",44        "layers.*.mlp.experts.down_proj": "grouped_gemm",45        "layers.*.mlp.experts.down_proj_bias": "grouped_gemm",46    }47 48    def __init__(49        self,50        num_hidden_layers: int = 36,51        num_local_experts: int = 128,52        vocab_size: int = 201088,53        hidden_size: int = 2880,54        intermediate_size: int = 2880,55        head_dim: int = 64,56        num_attention_heads: int = 64,57        num_key_value_heads: int = 8,58        sliding_window: int = 128,59        rope_theta: float = 150000.0,60        tie_word_embeddings=False,61        hidden_act: str = "silu",62        initializer_range: float = 0.02,63        max_position_embeddings=131072,64        rms_norm_eps: float = 1e-5,65        rope_scaling={66            "rope_type": "yarn",67            "factor": 32.0,68            "beta_fast": 32.0,69            "beta_slow": 1.0,70            "truncate": False,71            "original_max_position_embeddings": 4096,72        },73        attention_dropout: float = 0.0,74        num_experts_per_tok=4,75        router_aux_loss_coef: float = 0.9,76        output_router_logits=False,77        use_cache=True,78        layer_types=None,79        **kwargs,80    ):81        self.vocab_size = vocab_size82        self.hidden_size = hidden_size83        self.intermediate_size = intermediate_size84        self.num_hidden_layers = num_hidden_layers85        self.num_attention_heads = num_attention_heads86        self.num_local_experts = num_local_experts87        self.sliding_window = sliding_window88        self.num_experts_per_tok = num_experts_per_tok89        # for backward compatibility90        if num_key_value_heads is None:91            num_key_value_heads = num_attention_heads92 93        self.num_key_value_heads = num_key_value_heads94        self.hidden_act = hidden_act95        self.initializer_range = initializer_range96        self.rms_norm_eps = rms_norm_eps97        self.rope_theta = rope_theta98        self.rope_scaling = rope_scaling99        self.attention_dropout = attention_dropout100        self.head_dim = head_dim if head_dim is not None else self.hidden_size // self.num_attention_heads101        self.layer_types = layer_types102        if self.layer_types is None:103            self.layer_types = [104                "sliding_attention" if bool((i + 1) % 2) else "full_attention" for i in range(self.num_hidden_layers)105            ]106        layer_type_validation(self.layer_types, self.num_hidden_layers)107 108        self.attention_bias = True109        self.max_position_embeddings = max_position_embeddings110        self.router_aux_loss_coef = router_aux_loss_coef111        self.output_router_logits = output_router_logits112        self.use_cache = use_cache113 114        # Validate the correctness of rotary position embeddings parameters115        # BC: if there is a 'type' field, copy it it to 'rope_type'.116        if self.rope_scaling is not None and "type" in self.rope_scaling:117            self.rope_scaling["rope_type"] = self.rope_scaling["type"]118        rope_config_validation(self)119 120        super().__init__(121            tie_word_embeddings=tie_word_embeddings,122            **kwargs,123        )124 125 126__all__ = ["GptOssConfig"]127