OEvortex/HelpingAI-Vision

1import math2from typing import Optional3 4from transformers import PretrainedConfig5 6 7class PhiConfig(PretrainedConfig):8    """Phi configuration."""9 10    model_type = "phi-msft"11    attribute_map = {12        "max_position_embeddings": "n_positions",13        "hidden_size": "n_embd",14        "num_attention_heads": "n_head",15        "num_hidden_layers": "n_layer",16    }17 18    def __init__(19        self,20        vocab_size: int = 51200,21        n_positions: int = 2048,22        n_embd: int = 1024,23        n_layer: int = 20,24        n_inner: Optional[int] = None,25        n_head: int = 16,26        n_head_kv: Optional[int] = None,27        rotary_dim: Optional[int] = 32,28        activation_function: Optional[str] = "gelu_new",29        flash_attn: bool = False,30        flash_rotary: bool = False,31        fused_dense: bool = False,32        attn_pdrop: float = 0.0,33        embd_pdrop: float = 0.0,34        resid_pdrop: float = 0.0,35        layer_norm_epsilon: float = 1e-5,36        initializer_range: float = 0.02,37        tie_word_embeddings: bool = False,38        pad_vocab_size_multiple: int = 64,39        **kwargs40    ) -> None:41        self.vocab_size = int(math.ceil(vocab_size / pad_vocab_size_multiple) * pad_vocab_size_multiple)42        self.n_positions = n_positions43        self.n_embd = n_embd44        self.n_layer = n_layer45        self.n_inner = n_inner46        self.n_head = n_head47        self.n_head_kv = n_head_kv48        self.rotary_dim = min(rotary_dim, n_embd // n_head)49        self.activation_function = activation_function50        self.flash_attn = flash_attn51        self.flash_rotary = flash_rotary52        self.fused_dense = fused_dense53        self.attn_pdrop = attn_pdrop54        self.embd_pdrop = embd_pdrop55        self.resid_pdrop = resid_pdrop56        self.layer_norm_epsilon = layer_norm_epsilon57        self.initializer_range = initializer_range58 59        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)60