MiniMaxAI/MiniMax-Text-01

1""" MiniMaxText01 model configuration"""2 3from transformers.configuration_utils import PretrainedConfig4from transformers.utils import logging5 6 7logger = logging.get_logger(__name__)8 9 10class MiniMaxText01Config(PretrainedConfig):11    r"""12    This is the configuration class to store the configuration of a [`MiniMaxText01Model`]. It is used to instantiate an13    MiniMaxText01 model according to the specified arguments, defining the model architecture. Instantiating a configuration14    with the defaults will yield a similar configuration to that of the MiniMaxText01.15 16    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the17    documentation from [`PretrainedConfig`] for more information.18 19 20    Args:21        vocab_size (`int`, *optional*, defaults to 32000):22            Vocabulary size of the MiniMaxText01 model. Defines the number of different tokens that can be represented by the23            `inputs_ids` passed when calling [`MiniMaxText01Model`]24        hidden_size (`int`, *optional*, defaults to 4096):25            Dimension of the hidden representations.26        intermediate_size (`int`, *optional*, defaults to 14336):27            Dimension of the MLP representations.28        num_hidden_layers (`int`, *optional*, defaults to 32):29            Number of hidden layers in the Transformer encoder.30        num_attention_heads (`int`, *optional*, defaults to 32):31            Number of attention heads for each attention layer in the Transformer encoder.32        num_key_value_heads (`int`, *optional*, defaults to 8):33            This is the number of key_value heads that should be used to implement Grouped Query Attention. If34            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if35            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When36            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed37            by meanpooling all the original heads within that group. For more details checkout [this38            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.39        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):40            The non-linear activation function (function or string) in the decoder.41        max_position_embeddings (`int`, *optional*, defaults to `4096*32`):42            The maximum sequence length that this model might ever be used with. MiniMaxText01's sliding window attention43            allows sequence of up to 4096*32 tokens.44        initializer_range (`float`, *optional*, defaults to 0.02):45            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.46        rms_norm_eps (`float`, *optional*, defaults to 1e-05):47            The epsilon used by the rms normalization layers.48        use_cache (`bool`, *optional*, defaults to `True`):49            Whether or not the model should return the last key/values attentions (not used by all models). Only50            relevant if `config.is_decoder=True`.51        pad_token_id (`int`, *optional*):52            The id of the padding token.53        bos_token_id (`int`, *optional*, defaults to 1):54            The id of the "beginning-of-sequence" token.55        eos_token_id (`int`, *optional*, defaults to 2):56            The id of the "end-of-sequence" token.57        tie_word_embeddings (`bool`, *optional*, defaults to `False`):58            Whether the model's input and output word embeddings should be tied.59        rope_theta (`float`, *optional*, defaults to 1000000.0):60            The base period of the RoPE embeddings.61        sliding_window (`int`, *optional*):62            Sliding window attention window size. If not specified, will default to `4096`.63        attention_dropout (`float`, *optional*, defaults to 0.0):64            The dropout ratio for the attention probabilities.65        num_experts_per_tok (`int`, *optional*, defaults to 2):66            The number of experts to route per-token, can be also interpreted as the `top-k` routing67            parameter68        num_local_experts (`int`, *optional*, defaults to 8):69            Number of experts per Sparse MLP layer.70        output_router_logits (`bool`, *optional*, defaults to `False`):71            Whether or not the router logits should be returned by the model. Enabeling this will also72            allow the model to output the auxiliary loss. See [here]() for more details73        router_aux_loss_coef (`float`, *optional*, defaults to 0.001):74            The aux loss factor for the total loss.75        router_jitter_noise (`float`, *optional*, defaults to 0.0):76            Amount of noise to add to the router.77 78    ```python79    >>> from transformers import MiniMaxText01Model, MiniMaxText01Config80 81    >>> # Initializing a MiniMaxText01 style configuration82    >>> configuration = MiniMaxText01Config()83 84    >>> # Initializing a model from the MiniMaxText01 style configuration85    >>> model = MiniMaxText01Model(configuration)86 87    >>> # Accessing the model configuration88    >>> configuration = model.config89    ```"""90 91    model_type = "MiniMaxText01"92    keys_to_ignore_at_inference = ["past_key_values"]93 94    def __init__(95        self,96        vocab_size=32000,97        hidden_size=4096,98        intermediate_size=14336,99        num_hidden_layers=32,100        num_attention_heads=32,101        num_key_value_heads=8,102        hidden_act="silu",103        max_position_embeddings=4096 * 32,104        initializer_range=0.02,105        rms_norm_eps=1e-5,106        use_cache=True,107        pad_token_id=None,108        bos_token_id=None,109        eos_token_id=None,110        tie_word_embeddings=False,111        rope_theta=1e6,112        sliding_window=None,113        attention_dropout=0.0,114        num_experts_per_tok=2,115        num_local_experts=8,116        output_router_logits=False,117        router_aux_loss_coef=0.001,118        router_jitter_noise=0.0,119        **kwargs,120    ):121        self.vocab_size = vocab_size122        self.max_position_embeddings = max_position_embeddings123        self.hidden_size = hidden_size124        self.intermediate_size = intermediate_size125        self.num_hidden_layers = num_hidden_layers126        self.num_attention_heads = num_attention_heads127        self.sliding_window = sliding_window128 129        # for backward compatibility130        if num_key_value_heads is None:131            num_key_value_heads = num_attention_heads132 133        self.num_key_value_heads = num_key_value_heads134        self.hidden_act = hidden_act135        self.initializer_range = initializer_range136        self.rms_norm_eps = rms_norm_eps137        self.use_cache = use_cache138        self.rope_theta = rope_theta139        self.attention_dropout = attention_dropout140 141        self.num_experts_per_tok = num_experts_per_tok142        self.num_local_experts = num_local_experts143        self.output_router_logits = output_router_logits144        self.router_aux_loss_coef = router_aux_loss_coef145        self.router_jitter_noise = router_jitter_noise146        super().__init__(147            pad_token_id=pad_token_id,148            bos_token_id=bos_token_id,149            eos_token_id=eos_token_id,150            tie_word_embeddings=tie_word_embeddings,151            **kwargs,152        )153