WisdomShell/CodeShell-7B

1# coding=utf-82# Copyright 2023 WisdomShell Inc. All Rights Reserved.3 4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16# This code is based on Bigcode's GPTBigCode configuration. It has been modified from17# its original forms to accommodate minor architectural differences compared to 18# GPTBigCode Configuration that trained the model.19 20# Copyright 2023 The BigCode team and HuggingFace Inc. team.21#22# Licensed under the Apache License, Version 2.0 (the "License");23# you may not use this file except in compliance with the License.24# You may obtain a copy of the License at25#26#     http://www.apache.org/licenses/LICENSE-2.027#28# Unless required by applicable law or agreed to in writing, software29# distributed under the License is distributed on an "AS IS" BASIS,30# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.31# See the License for the specific language governing permissions and32# limitations under the License.33""" Shell configuration"""34 35from transformers.configuration_utils import PretrainedConfig36from transformers.utils import logging37 38 39logger = logging.get_logger(__name__)40 41 42class CodeShellConfig(PretrainedConfig):43    """44    This is the configuration class to store the configuration of a [`CodeShellModel`]. It is used to instantiate a45    CodeShell model according to the specified arguments, defining the model architecture.46 47    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the48    documentation from [`PretrainedConfig`] for more information.49 50    Args:51        vocab_size (`int`, *optional*, defaults to 50257):52            Vocabulary size of the GPT-2 model. Defines the number of different tokens that can be represented by the53            `inputs_ids` passed when calling [`ShellModel`].54        n_positions (`int`, *optional*, defaults to 1024):55            The maximum sequence length that this model might ever be used with. Typically set this to something large56            just in case (e.g., 512 or 1024 or 2048).57        n_embd (`int`, *optional*, defaults to 768):58            Dimensionality of the embeddings and hidden states.59        n_layer (`int`, *optional*, defaults to 12):60            Number of hidden layers in the Transformer encoder.61        n_head (`int`, *optional*, defaults to 12):62            Number of attention heads for each attention layer in the Transformer encoder.63        n_inner (`int`, *optional*, defaults to None):64            Dimensionality of the inner feed-forward layers. `None` will set it to 4 times n_embd65        activation_function (`str`, *optional*, defaults to `"gelu_pytorch_tanh"`):66            Activation function, to be selected in the list `["relu", "silu", "gelu", "tanh", "gelu_new",67            "gelu_pytorch_tanh"]`.68        resid_pdrop (`float`, *optional*, defaults to 0.1):69            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.70        embd_pdrop (`float`, *optional*, defaults to 0.1):71            The dropout ratio for the embeddings.72        attn_pdrop (`float`, *optional*, defaults to 0.1):73            The dropout ratio for the attention.74        layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):75            The epsilon to use in the layer normalization layers.76        initializer_range (`float`, *optional*, defaults to 0.02):77            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.78        scale_attn_weights (`bool`, *optional*, defaults to `True`):79            Scale attention weights by dividing by sqrt(hidden_size)..80        use_cache (`bool`, *optional*, defaults to `True`):81            Whether or not the model should return the last key/values attentions (not used by all models).82        attention_softmax_in_fp32 (`bool`, *optional*, defaults to `True`):83            Whether to call the fused softmax in float32.84        scale_attention_softmax_in_fp32 (`bool`, *optional*, defaults to `True`):85            Whether to scale the attention softmax in float32.86        attention_type (`bool`, *optional*, defaults to `True`):87            Whether to use Multi-Query Attion (`True`) or Multi-Head Attention (`False`).88    Example:89 90    ```python91    >>> from configuration_codeshell import CodeShellConfig92    >>> from modeling_codeshell import CodeShellForCausalLM93 94    >>> # Initializing a CodeShell configuration95    >>> configuration = CodeShellConfig()96 97    >>> # Initializing a model (with random weights) from the configuration98    >>> model = CodeShellForCausalLM(configuration)99 100    >>> # Accessing the model configuration101    >>> configuration = model.config102    ```"""103 104    model_type = "codeshell"105    keys_to_ignore_at_inference = ["past_key_values"]106    attribute_map = {107        "hidden_size": "n_embd",108        "max_position_embeddings": "n_positions",109        "num_attention_heads": "n_head",110        "num_hidden_layers": "n_layer",111    }112 113    def __init__(114        self,115        vocab_size=70144,116        n_positions=8192,117        n_embd=4096,118        n_layer=42,119        n_head=32,120        n_inner=None,121        activation_function="gelu_pytorch_tanh",122        resid_pdrop=0.1,123        embd_pdrop=0.1,124        attn_pdrop=0.1,125        layer_norm_epsilon=1e-5,126        initializer_range=0.02,127        scale_attn_weights=True,128        use_cache=True,129        bos_token_id=70000,130        eos_token_id=70000,131        attention_softmax_in_fp32=True,132        scale_attention_softmax_in_fp32=True,133        group_query_attention=True,134        num_query_groups=1,135        position_embedding_type="learned_absolute",136        rope_scaling=None,137        **kwargs,138    ):139        self.vocab_size = vocab_size140        self.n_positions = n_positions141        self.n_embd = n_embd142        self.n_layer = n_layer143        self.n_head = n_head144        self.n_inner = n_inner145        self.activation_function = activation_function146        self.resid_pdrop = resid_pdrop147        self.embd_pdrop = embd_pdrop148        self.attn_pdrop = attn_pdrop149        self.layer_norm_epsilon = layer_norm_epsilon150        self.initializer_range = initializer_range151        self.scale_attn_weights = scale_attn_weights152        self.use_cache = use_cache153        self.attention_softmax_in_fp32 = attention_softmax_in_fp32154        self.scale_attention_softmax_in_fp32 = scale_attention_softmax_in_fp32155        self.group_query_attention = group_query_attention156        self.num_query_groups = num_query_groups157        self.position_embedding_type = position_embedding_type158        self.rope_scaling = rope_scaling159        assert self.position_embedding_type in [160            "learned_absolute", "rope"161        ], "position_embedding_type must be one of ['learned_absolute', 'rope']"162        163        self.bos_token_id = bos_token_id164        self.eos_token_id = eos_token_id165 166        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)167