numind/NuExtract-large

1# coding=utf-82# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.3# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9#     http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16from typing import Any, Dict, List, Optional, Union17 18from transformers.configuration_utils import PretrainedConfig19from transformers.utils import logging20 21from functools import cached_property22 23""" Phi3Small model configuration """24logger = logging.get_logger(__name__)25 26 27def next_mult(x, y):28    return (x + y - 1) // y * y29 30class Phi3SmallConfig(PretrainedConfig):31    """32    This is the configuration class to store the configuration of a `Phi3Small` model. It is used to33    instantiate a Phi-3-small model according to the specified arguments, defining the model architecture. 34    Instantiating a configuration with the defaults will yield a similar configuration to that of the Phi-3-small35    [phi3](https://arxiv.org/pdf/2404.14219) architecture.36 37    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the38    documentation from [`PretrainedConfig`] for more information.39 40 41    Args:42        vocab_size (`int`, *optional*, defaults to 100352):43            Vocabulary size of the Phi3Small model. Defines the number of different tokens that can be represented by the44            `inputs_ids` passed when calling `Phi3Small`.45        max_position_embeddings (`int`, *optional*, defaults to 8192):46            The maximum sequence length that this model might safely be used with.47        rope_embedding_base (`float`, *optional*, defaults to 10^6):48            The base value for the RoPE (Relative Position Encoding) embedding.49        rope_position_scale (`float`, *optional*, defaults to 1.0):50            The scale factor for the RoPE position encoding.51        rope_scaling (`Optional[Dict[str, Union[float, List[float], int]]]`, *optional*, defaults to None):52            The scaling configuration used for LongRoPE.53        hidden_size (`int`, *optional*, defaults to 4096):54            The size of the hidden layers in the model.55        num_hidden_layers (`int`, *optional*, defaults to 32):56            The number of layers in the model.57        num_attention_heads (`int`, *optional*, defaults to 32):58            The number of query heads in the model.59        num_key_value_heads (`int`, *optional*, defaults to 8):60            The number of key-value heads in the model.61        hidden_act (`str`, *optional*, defaults to "gegelu"):62            The activation function used in the model.63        gegelu_limit (`float`, *optional*, defaults to 20.0):64            The limit value for the GELU activation function (for numerical stability).65        gegelu_pad_to_256 (`bool`, *optional*, defaults to True):66            Whether to pad the intermediate size to a multiple of 256 (for faster matmul ops).67        ff_dim_multiplier (`Optional[int]`, *optional*, defaults to None):68            The dimension multiplier for the feed-forward layers.69        ff_intermediate_size (`Optional[int]`, *optional*, defaults to 14336):70            The intermediate size for the feed-forward layers.71            One of `ff_dim_multiplier` or `ff_intermediate_size` must be specified.72        blocksparse_homo_head_pattern (`bool`, *optional*, defaults to False):73            Whether to use a homogeneous head pattern for block-sparse attention.74        blocksparse_block_size (`int`, *optional*, defaults to 64):75            The block size for block-sparse attention.76        blocksparse_num_local_blocks (`int`, *optional*, defaults to 16):77            The number of local blocks for block-sparse attention.78            The local window used in blocksparse equals `blocksparse_num_local_blocks * blocksparse_block_size`79        blocksparse_vert_stride (`int`, *optional*, defaults to 8):80            The vertical stride for block-sparse attention.81        blocksparse_triton_kernel_block_size (`int`, *optional*, defaults to 64):82            The kernel block size for block-sparse attention.83        dense_attention_every_n_layers (`Optional[int]`, *optional*, defaults to 2):84            The frequency of all dense attention layers in the model85        embedding_dropout_prob (`float`, *optional*, defaults to 0.1):86            The dropout probability for the embedding layer.87        attention_dropout_prob (`float`, *optional*, defaults to 0.0):88            The dropout probability for the attention layers.89        ffn_dropout_prob (`float`, *optional*, defaults to 0.1):90            The dropout probability for the feed-forward layers.91        layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):92            The epsilon value for layer normalization.93        initializer_range (`float`, *optional*, defaults to 0.02):94            The range for weight initialization.95        mup_use_scaling (`bool`, *optional*, defaults to True):96            Whether to use scaling for MuP parameters (see: https://arxiv.org/abs/2203.03466).97        mup_width_multiplier (`bool`, *optional*, defaults to 8.0):98            The width multiplier for MuP.99        mup_embedding_multiplier (`bool`, *optional*, defaults to 10.0):100            The embedding multiplier for MuP.101        mup_attn_multiplier (`bool`, *optional*, defaults to 1.0):102            The attention multiplier for MuP.103        use_cache (`bool`, *optional*, defaults to True):104            Whether to use cache for the model.105        bos_token_id (`int`, *optional*, defaults to 100257):106            The token ID for the beginning of sentence.107        eos_token_id (`int`, *optional*, defaults to 100257):108            The token ID for the end of sentence.109        reorder_and_upcast_attn (`bool`, *optional*, defaults to False):110            Whether to reorder and upcast attention.111        pad_sequence_to_multiple_of_64 (`bool`, *optional*, defaults to True):112            Whether to pad the sequence length to a multiple of 64.113        **kwargs:114            Additional keyword arguments.115 116    Example:117 118    ```python119    >>> from transformers import Phi3SmallConfig, Phi3SmallModel120 121    >>> # Initializing a Phi3Small configuration122    >>> configuration = Phi3SmallConfig()123 124    >>> # Initializing a model (with random weights) from the configuration125    >>> model = Phi3SmallModel(configuration)126 127    >>> # Accessing the model configuration128    >>> configuration = model.config129    ```130    """131 132    model_type = "phi3small"133    keys_to_ignore_at_inference = ["past_key_values"]134    135 136    def __init__(137        self,138        # General information about the model139        vocab_size: int =100352,140        max_position_embeddings: int = 8192,141        # RoPE Related Parameters142        rope_embedding_base: float = 10**6,143        rope_position_scale: float = 1.0,144        rope_scaling: Optional[Dict[str, Union[float, List[float], int]]] = None,145        # General Model Parameters146        hidden_size: int = 4096,147        num_hidden_layers: int = 32,148        # KV Shared Attention Configurations149        num_attention_heads: int = 32,150        num_key_value_heads: int = 8,151        # GEGELU Related Parameters152        hidden_act: str = "gegelu",153        gegelu_limit: float = 20.0,154        gegelu_pad_to_256: bool = True,155        ff_dim_multiplier: Optional[int] = None,156        ff_intermediate_size: Optional[int] = 14336,157        # Block Sparse Attention Parameters158        blocksparse_homo_head_pattern: bool = False,159        blocksparse_block_size: int = 64,160        blocksparse_num_local_blocks: int = 16,161        blocksparse_vert_stride: int = 8,162        blocksparse_triton_kernel_block_size: int = 64,163        # Frequency of block-sparsity164        dense_attention_every_n_layers: Optional[int] = 2,165        # Reegularization parameters166        embedding_dropout_prob: float =0.1,167        attention_dropout_prob: float = 0.0,168        ffn_dropout_prob: float = 0.1,169        layer_norm_epsilon=1e-5,170        initializer_range=0.02,171        # MuP parameters172        mup_use_scaling: bool = True,173        mup_width_multiplier: bool = 8.0,174        mup_embedding_multiplier: bool = 10.0,175        mup_attn_multiplier: bool =1.0,176        use_cache=True,177        # The model does not have a bos token id178        # However, in order for some of the downstream libraries to not break179        # we set this to be the same as the eos_token_id180        bos_token_id: int = 100257,181        eos_token_id: int = 100257,182        reorder_and_upcast_attn=False,183        # Configuration to pad sequence length to a multiple of 64184        pad_sequence_to_multiple_of_64: bool = True,185        **kwargs,186    ):187        self.vocab_size = vocab_size188        self.max_position_embeddings = max_position_embeddings189        self.rope_embedding_base = rope_embedding_base190        self.rope_position_scale = rope_position_scale191        self.rope_scaling = rope_scaling192        self.hidden_size = hidden_size193        # QK Shared Attention194        self.num_hidden_layers = num_hidden_layers195        self.num_attention_heads = num_attention_heads196        self.num_key_value_heads = num_key_value_heads197        # Block Sparse Attention Pattern198        self.blocksparse_homo_head_pattern = blocksparse_homo_head_pattern199        self.blocksparse_block_size = blocksparse_block_size200        self.blocksparse_num_local_blocks = blocksparse_num_local_blocks201        self.blocksparse_vert_stride = blocksparse_vert_stride202        self.blocksparse_triton_kernel_block_size = blocksparse_triton_kernel_block_size203        # Frequency of block sparsity204        self.dense_attention_every_n_layers = dense_attention_every_n_layers205        # Activation function206        self.hidden_act = hidden_act207        self.gegelu_limit = gegelu_limit208        self.gegelu_pad_to_256 = gegelu_pad_to_256209        self.ff_dim_multiplier = ff_dim_multiplier210        self.ff_intermediate_size = ff_intermediate_size211        if self.ff_dim_multiplier is None and self.ff_intermediate_size is None:212            raise ValueError(f"Cannot have both {self.ff_dim_multiplier} and {self.ff_intermediate_size} as None")213        if self.ff_dim_multiplier is not None and self.ff_intermediate_size is not None:214            raise ValueError(f"Cannot specify both {self.ff_dim_multiplier} and {self.ff_intermediate_size}.")215        # General regularization216        self.embedding_dropout_prob = embedding_dropout_prob217        self.attention_dropout_prob = attention_dropout_prob218        self.ffn_dropout_prob = ffn_dropout_prob219        self.layer_norm_epsilon = layer_norm_epsilon220        self.initializer_range = initializer_range221        # MuP parameters222        self.mup_use_scaling = mup_use_scaling223        self.mup_width_multiplier = mup_width_multiplier224        self.mup_embedding_multiplier = mup_embedding_multiplier225        self.mup_attn_multiplier = mup_attn_multiplier226        self.use_cache = use_cache227 228        self.reorder_and_upcast_attn = reorder_and_upcast_attn229        self.pad_sequence_to_multiple_of_64 = pad_sequence_to_multiple_of_64230 231        self.bos_token_id = bos_token_id232        self.eos_token_id = eos_token_id233 234        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)235 236    @cached_property237    def dummy_token_indices(self) -> List[int]:238        # Importing here to avoid circular imports239        from .tokenization_phi3_small import Phi3SmallTokenizer240        tokenizer = Phi3SmallTokenizer()241        return tokenizer.dummy_token_indices242 243    @property244    def intermediate_size(self) -> int:245        if self.ff_intermediate_size is not None:246            return self.ff_intermediate_size247        intermediate_size = (self.ff_dim_multiplier) * (self.hidden_size // 3) * 2248        if self.gegelu_pad_to_256:249            intermediate_size = next_mult(intermediate_size, 256)250        return intermediate_size251