Aluode/PerceptionLabPortable

1# coding=utf-82# Copyright 2023, HuggingFace Inc.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""NLLB-MoE model configuration"""16 17from ...configuration_utils import PretrainedConfig18from ...utils import logging19 20 21logger = logging.get_logger(__name__)22 23 24class NllbMoeConfig(PretrainedConfig):25    r"""26    This is the configuration class to store the configuration of a [`NllbMoeModel`]. It is used to instantiate an27    NLLB-MoE model according to the specified arguments, defining the model architecture. Instantiating a configuration28    with the defaults will yield a similar configuration to that of the NLLB-MoE29    [facebook/nllb-moe-54b](https://huggingface.co/facebook/nllb-moe-54b) architecture.30 31    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the32    documentation from [`PretrainedConfig`] for more information.33 34 35    Args:36        vocab_size (`int`, *optional*, defaults to 50265):37            Vocabulary size of the NllbMoe model. Defines the number of different tokens that can be represented by the38            `inputs_ids` passed when calling [`NllbMoeModel`] or39        d_model (`int`, *optional*, defaults to 1024):40            Dimensionality of the layers and the pooler layer.41        encoder_layers (`int`, *optional*, defaults to 12):42            Number of encoder layers.43        decoder_layers (`int`, *optional*, defaults to 12):44            Number of decoder layers.45        encoder_attention_heads (`int`, *optional*, defaults to 16):46            Number of attention heads for each attention layer in the Transformer encoder.47        decoder_attention_heads (`int`, *optional*, defaults to 16):48            Number of attention heads for each attention layer in the Transformer decoder.49        decoder_ffn_dim (`int`, *optional*, defaults to 4096):50            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.51        encoder_ffn_dim (`int`, *optional*, defaults to 4096):52            Dimensionality of the "intermediate" (often named feed-forward) layer in encoder.53        activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):54            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,55            `"relu"`, `"silu"` and `"gelu_new"` are supported.56        dropout (`float`, *optional*, defaults to 0.1):57            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.58        attention_dropout (`float`, *optional*, defaults to 0.0):59            The dropout ratio for the attention probabilities.60        activation_dropout (`float`, *optional*, defaults to 0.0):61            The dropout ratio for activations inside the fully connected layer.62        classifier_dropout (`float`, *optional*, defaults to 0.0):63            The dropout ratio for classifier.64        max_position_embeddings (`int`, *optional*, defaults to 1024):65            The maximum sequence length that this model might ever be used with. Typically set this to something large66            just in case (e.g., 512 or 1024 or 2048).67        init_std (`float`, *optional*, defaults to 0.02):68            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.69        encoder_layerdrop (`float`, *optional*, defaults to 0.0):70            The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://huggingface.co/papers/1909.11556)71            for more details.72        decoder_layerdrop (`float`, *optional*, defaults to 0.0):73            The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://huggingface.co/papers/1909.11556)74            for more details.75        second_expert_policy ( `str`, *optional*, default to `"all"`):76            The policy used for the sampling the probability of being sampled to a second expert for each token.77        normalize_router_prob_before_dropping (`bool`, *optional*, defaults to `True`):78            Whether or not to normalize the router probabilities before applying a mask based on the experts capacity79            (capacity dropping).80        batch_prioritized_routing (`bool`, *optional*, defaults to `True`):81            Whether or not to orders the tokens by their router probabilities before capacity dropping. This means that82            the tokens that have the highest probabilities will be routed before other tokens that might be further in83            the sequence.84        moe_eval_capacity_token_fraction (`float`, *optional*, defaults to 1.0):85            Fraction of tokens as capacity during validation, if set to negative, uses the same as training. Should be86            in range: (0.0, 1.0].87        num_experts (`int`, *optional*, defaults to 128):88            Number of experts for each NllbMoeSparseMlp layer.89        expert_capacity (`int`, *optional*, defaults to 64):90            Number of tokens that can be stored in each expert.91        encoder_sparse_step (`int`, *optional*, defaults to 4):92            Frequency of the sparse layers in the encoder. 4 means that one out of 4 layers will be sparse.93        decoder_sparse_step (`int`, *optional*, defaults to 4):94            Frequency of the sparse layers in the decoder. 4 means that one out of 4 layers will be sparse.95        router_dtype (`str`, *optional*, default to `"float32"`):96            The `dtype` used for the routers. It is preferable to keep the `dtype` to `"float32"` as specified in the97            *selective precision* discussion in [the paper](https://huggingface.co/papers/2101.03961).98        router_ignore_padding_tokens (`bool`, *optional*, defaults to `False`):99            Whether to ignore padding tokens when routing. if `False`, the padding tokens are not routed to any100            experts.101        router_bias (`bool`, *optional*, defaults to `False`):102            Whether or not the classifier of the router should have a bias.103        moe_token_dropout (`float`, *optional*, default to 0.2):104            Masking rate for MoE expert output masking (EOM), which is implemented via a Dropout2d on the expert105            outputs.106        output_router_logits (`bool`, *optional*, defaults to `False`):107            Whether or not to return the router logits. Only set to `True` to get the auxiliary loss when training.108        use_cache (`bool`, *optional*, defaults to `True`):109            Whether or not the model should return the last key/values attentions (not used by all models).110 111    Example:112 113    ```python114    >>> from transformers import NllbMoeModel, NllbMoeConfig115 116    >>> # Initializing a NllbMoe facebook/nllb-moe-54b style configuration117    >>> configuration = NllbMoeConfig()118 119    >>> # Initializing a model from the facebook/nllb-moe-54b style configuration120    >>> model = NllbMoeModel(configuration)121 122    >>> # Accessing the model configuration123    >>> configuration = model.config124    ```"""125 126    model_type = "nllb-moe"127    keys_to_ignore_at_inference = ["past_key_values"]128    attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}129 130    def __init__(131        self,132        vocab_size=128112,133        max_position_embeddings=1024,134        encoder_layers=12,135        encoder_ffn_dim=4096,136        encoder_attention_heads=16,137        decoder_layers=12,138        decoder_ffn_dim=4096,139        decoder_attention_heads=16,140        encoder_layerdrop=0.05,141        decoder_layerdrop=0.05,142        use_cache=True,143        is_encoder_decoder=True,144        activation_function="relu",145        d_model=1024,146        dropout=0.1,147        attention_dropout=0.1,148        activation_dropout=0.0,149        init_std=0.02,150        decoder_start_token_id=2,151        scale_embedding=True,152        router_bias=False,153        router_dtype="float32",154        router_ignore_padding_tokens=False,155        num_experts=128,156        expert_capacity=64,157        encoder_sparse_step=4,158        decoder_sparse_step=4,159        router_z_loss_coef=0.001,160        router_aux_loss_coef=0.001,161        second_expert_policy="all",162        normalize_router_prob_before_dropping=False,163        batch_prioritized_routing=False,164        moe_eval_capacity_token_fraction=1.0,165        moe_token_dropout=0.2,166        pad_token_id=1,167        bos_token_id=0,168        eos_token_id=2,169        output_router_logits=False,170        **kwargs,171    ):172        self.vocab_size = vocab_size173        self.max_position_embeddings = max_position_embeddings174        self.d_model = d_model175        self.encoder_ffn_dim = encoder_ffn_dim176        self.encoder_layers = encoder_layers177        self.encoder_attention_heads = encoder_attention_heads178        self.decoder_ffn_dim = decoder_ffn_dim179        self.decoder_layers = decoder_layers180        self.decoder_attention_heads = decoder_attention_heads181        self.dropout = dropout182        self.attention_dropout = attention_dropout183        self.activation_dropout = activation_dropout184        self.activation_function = activation_function185        self.init_std = init_std186        self.encoder_layerdrop = encoder_layerdrop187        self.decoder_layerdrop = decoder_layerdrop188        self.use_cache = use_cache189        self.num_hidden_layers = encoder_layers190        self.scale_embedding = scale_embedding  # scale factor will be sqrt(d_model) if True191        self.router_z_loss_coef = router_z_loss_coef192        self.router_aux_loss_coef = router_aux_loss_coef193        self.decoder_sparse_step = decoder_sparse_step194        self.encoder_sparse_step = encoder_sparse_step195        self.num_experts = num_experts196        self.expert_capacity = expert_capacity197        self.router_bias = router_bias198        if router_dtype not in ["float32", "float16", "bfloat16"]:199            raise ValueError(f"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}")200        self.router_dtype = router_dtype201 202        self.router_ignore_padding_tokens = router_ignore_padding_tokens203        self.batch_prioritized_routing = batch_prioritized_routing204        self.second_expert_policy = second_expert_policy205        self.normalize_router_prob_before_dropping = normalize_router_prob_before_dropping206        self.moe_eval_capacity_token_fraction = moe_eval_capacity_token_fraction207        self.moe_token_dropout = moe_token_dropout208        self.output_router_logits = output_router_logits209        super().__init__(210            pad_token_id=pad_token_id,211            bos_token_id=bos_token_id,212            eos_token_id=eos_token_id,213            is_encoder_decoder=is_encoder_decoder,214            decoder_start_token_id=decoder_start_token_id,215            **kwargs,216        )217 218 219__all__ = ["NllbMoeConfig"]220