jdopensource/JoyAI-LLM-Flash

1# coding=utf-82# Copyright 2025 bzantium and the HuggingFace Inc. team. All rights reserved.3#4# This code is based on the DeepSeekV3 implementations from the DeepSeek AI team. (https://huggingface.co/deepseek-ai/DeepSeek-V3)5 6# Licensed under the Apache License, Version 2.0 (the "License");7# you may not use this file except in compliance with the License.8# You may obtain a copy of the License at9#10#     http://www.apache.org/licenses/LICENSE-2.011#12# Unless required by applicable law or agreed to in writing, software13# distributed under the License is distributed on an "AS IS" BASIS,14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.15# See the License for the specific language governing permissions and16# limitations under the License.17"""DeepSeekV3 model configuration"""18 19from transformers.configuration_utils import PretrainedConfig20from transformers.modeling_rope_utils import rope_config_validation21 22 23DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}24 25 26class DeepseekV3Config(PretrainedConfig):27    r"""28    This is the configuration class to store the configuration of a [`DeepseekV3Model`]. It is used to instantiate an DeepSeek29    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the30    defaults will yield a similar configuration to that of the DeepSeek-V3.31    e.g. [bzantium/tiny-deepseek-v3](https://huggingface.co/bzantium/tiny-deepseek-v3)32    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the33    documentation from [`PretrainedConfig`] for more information.34 35 36    Args:37        vocab_size (`int`, *optional*, defaults to 129280):38            Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the39            `inputs_ids` passed when calling [`DeepseekV3Model`]40        hidden_size (`int`, *optional*, defaults to 7168):41            Dimension of the hidden representations.42        intermediate_size (`int`, *optional*, defaults to 18432):43            Dimension of the MLP representations.44        moe_intermediate_size (`int`, *optional*, defaults to 2048):45            Dimension of the MoE representations.46        num_hidden_layers (`int`, *optional*, defaults to 61):47            Number of hidden layers in the Transformer decoder.48        num_attention_heads (`int`, *optional*, defaults to 128):49            Number of attention heads for each attention layer in the Transformer decoder.50        num_key_value_heads (`int`, *optional*, defaults to 128):51            This is the number of key_value heads that should be used to implement Grouped Query Attention. If52            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if53            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When54            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed55            by meanpooling all the original heads within that group. For more details checkout [this56            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to57            `num_attention_heads`.58        n_shared_experts (`int`, *optional*, defaults to 1):59            Number of shared experts.60        n_routed_experts (`int`, *optional*, defaults to 256):61            Number of routed experts.62        routed_scaling_factor (`float`, *optional*, defaults to 2.5):63            Scaling factor or routed experts.64        kv_lora_rank (`int`, *optional*, defaults to 512):65            Rank of the LoRA matrices for key and value projections.66        q_lora_rank (`int`, *optional*, defaults to 1536):67            Rank of the LoRA matrices for query projections.68        qk_rope_head_dim (`int`, *optional*, defaults to 64):69            Dimension of the query/key heads that use rotary position embeddings.70        v_head_dim (`int`, *optional*, defaults to 128):71            Dimension of the value heads.72        qk_nope_head_dim (`int`, *optional*, defaults to 128):73            Dimension of the query/key heads that don't use rotary position embeddings.74        n_group (`int`, *optional*, defaults to 8):75            Number of groups for routed experts.76        topk_group (`int`, *optional*, defaults to 4):77            Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).78        num_experts_per_tok (`int`, *optional*, defaults to 8):79            Number of selected experts, None means dense model.80        first_k_dense_replace (`int`, *optional*, defaults to 3):81            Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).82                                                            \--k dense layers--/83        norm_topk_prob (`bool`, *optional*, defaults to `True`):84            Whether to normalize the weights of the routed experts.85        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):86            The non-linear activation function (function or string) in the decoder.87        max_position_embeddings (`int`, *optional*, defaults to 4096):88            The maximum sequence length that this model might ever be used with.89        initializer_range (`float`, *optional*, defaults to 0.02):90            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.91        rms_norm_eps (`float`, *optional*, defaults to 1e-06):92            The epsilon used by the rms normalization layers.93        use_cache (`bool`, *optional*, defaults to `True`):94            Whether or not the model should return the last key/values attentions (not used by all models). Only95            relevant if `config.is_decoder=True`.96        pad_token_id (`int`, *optional*):97            Padding token id.98        bos_token_id (`int`, *optional*, defaults to 0):99            Beginning of stream token id.100        eos_token_id (`int`, *optional*, defaults to 1):101            End of stream token id.102        pretraining_tp (`int`, *optional*, defaults to 1):103            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this104            document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is105            necessary to ensure exact reproducibility of the pretraining results. Please refer to [this106            issue](https://github.com/pytorch/pytorch/issues/76232).107        tie_word_embeddings (`bool`, *optional*, defaults to `False`):108            Whether to tie weight embeddings109        rope_theta (`float`, *optional*, defaults to 10000.0):110            The base period of the RoPE embeddings.111        rope_scaling (`Dict`, *optional*):112            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling113            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is114            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update115            `max_position_embeddings` to the expected new maximum.116        rope_interleave (`bool`, *optional*, defaults to `True`):117            Whether to interleave the rotary position embeddings.118        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):119            Whether to use a bias in the query, key, value and output projection layers during self-attention.120        attention_dropout (`float`, *optional*, defaults to 0.0):121            The dropout ratio for the attention probabilities.122 123    ```python124    >>> from transformers import DeepseekV3Model, DeepseekV3Config125 126    >>> # Initializing a Deepseek-V3 style configuration127    >>> configuration = DeepseekV3Config()128 129    >>> # Accessing the model configuration130    >>> configuration = model.config131    ```"""132 133    model_type = "deepseek_v3"134    keys_to_ignore_at_inference = ["past_key_values"]135    base_model_tp_plan = {  # TODO: only replicate attention layers when > first_k_dense_replace136        "layers.*.mlp.experts.*.gate_proj": "local_colwise",137        "layers.*.mlp.experts.*.up_proj": "local_colwise",138        "layers.*.mlp.experts.*.down_proj": "local_rowwise",139        "layers.*.mlp.experts.*": "local",  # each expert is wrapped in a module list140        "layers.*.mlp.shared_experts.gate_proj": "local_colwise",141        "layers.*.mlp.shared_experts.up_proj": "local_colwise",142        "layers.*.mlp.shared_experts.down_proj": "local_rowwise",143        "layers.*.mlp.shared_experts": "local",144        "layers.*.mlp.gate_proj": "local_colwise",145        "layers.*.mlp.up_proj": "local_colwise",146        "layers.*.mlp.down_proj": "local_rowwise",147        "layers.*.mlp": "gather",  # This is the only moment where results are gathered148    }149    base_model_pp_plan = {150        "embed_tokens": (["input_ids"], ["inputs_embeds"]),151        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),152        "norm": (["hidden_states"], ["hidden_states"]),153    }154 155    def __init__(156        self,157        vocab_size=129280,158        hidden_size=7168,159        intermediate_size=18432,160        moe_intermediate_size=2048,161        num_hidden_layers=61,162        num_attention_heads=128,163        num_key_value_heads=128,164        n_shared_experts=1,165        n_routed_experts=256,166        routed_scaling_factor=2.5,167        kv_lora_rank=512,168        q_lora_rank=1536,169        qk_rope_head_dim=64,170        v_head_dim=128,171        qk_nope_head_dim=128,172        n_group=8,173        topk_group=4,174        num_experts_per_tok=8,175        first_k_dense_replace=3,176        norm_topk_prob=True,177        hidden_act="silu",178        max_position_embeddings=4096,179        initializer_range=0.02,180        rms_norm_eps=1e-6,181        use_cache=True,182        pad_token_id=None,183        bos_token_id=0,184        eos_token_id=1,185        pretraining_tp=1,186        tie_word_embeddings=False,187        rope_theta=10000.0,188        rope_scaling=None,189        rope_interleave=True,190        attention_bias=False,191        attention_dropout=0.0,192        **kwargs,193    ):194        self.vocab_size = vocab_size195        self.max_position_embeddings = max_position_embeddings196        self.hidden_size = hidden_size197        self.intermediate_size = intermediate_size198        self.moe_intermediate_size = moe_intermediate_size199        self.num_hidden_layers = num_hidden_layers200        self.num_attention_heads = num_attention_heads201        self.n_shared_experts = n_shared_experts202        self.n_routed_experts = n_routed_experts203        self.routed_scaling_factor = routed_scaling_factor204        self.kv_lora_rank = kv_lora_rank205        self.q_lora_rank = q_lora_rank206        self.qk_rope_head_dim = qk_rope_head_dim207        self.v_head_dim = v_head_dim208        self.qk_nope_head_dim = qk_nope_head_dim209        self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim210        self.head_dim = qk_rope_head_dim211        self.n_group = n_group212        self.topk_group = topk_group213        self.num_experts_per_tok = num_experts_per_tok214        self.first_k_dense_replace = first_k_dense_replace215        self.norm_topk_prob = norm_topk_prob216        self.rope_interleave = rope_interleave217 218        # for backward compatibility219        if num_key_value_heads is None:220            num_key_value_heads = num_attention_heads221 222        self.num_key_value_heads = num_key_value_heads223        self.hidden_act = hidden_act224        self.initializer_range = initializer_range225        self.rms_norm_eps = rms_norm_eps226        self.pretraining_tp = pretraining_tp227        self.use_cache = use_cache228        self.rope_theta = rope_theta229        self.rope_scaling = rope_scaling230        self.attention_bias = attention_bias231        self.attention_dropout = attention_dropout232        # Validate the correctness of rotary position embeddings parameters233        # BC: if there is a 'type' field, copy it it to 'rope_type'.234        if self.rope_scaling is not None and "type" in self.rope_scaling:235            self.rope_scaling["rope_type"] = self.rope_scaling["type"]236        rope_config_validation(self)237 238        super().__init__(239            pad_token_id=pad_token_id,240            bos_token_id=bos_token_id,241            eos_token_id=eos_token_id,242            tie_word_embeddings=tie_word_embeddings,243            **kwargs,244        )245 246 247__all__ = ["DeepseekV3Config"]248