chris0809/tinyLLM-0.51B-VLM

1from transformers import PretrainedConfig
2from transformers import AutoTokenizer
3
4class Config(PretrainedConfig):
5    model_type = "tinyLLM"
6
7    def __init__(self,
8                 eos_token_id: int | None = 73440,
9                 pad_token_id: int | None = 73440,
10                use_tcn = False,
11                tcn_layers = [],
12                 num_query_tokens=128,
13                vocab_size=73448,
14                hidden_size=1280,
15                num_hidden_layers=24,
16                num_attention_heads=20,
17                num_key_value_heads=4,
18                max_position_embeddings=8192,
19                RoPE_base=1e4,
20                rope_type="yarn",#["default","yarn","dynamic"]
21                dropout=0.00,
22                rms_norm_eps=1e-5,
23                qkrms_norm_eps=1e-5,
24                use_qk_norm=True,
25                kv_cache_dtype="auto",
26                use_moe=False,
27                adapter_type="none",
28                use_swiGLU=True,
29                bos_token_id=None,
30                use_ssm=False,
31                ssm_layers=[],#[4,11,19]
32                embeddingdropout=0.0,
33                learnable_temp=True,
34                use_affine=True,
35                use_sampleattion=False,
36                eso_loss_radio=1.0,
37                train_maxlength=8192,
38                drop_high_loss=0.0,
39                mlp_ratio_front: float = 3.5,
40                mlp_ratio_mid: float = 4.0,
41                mlp_ratio_back: float = 4.5,
42                mlp_mid_start: int | None = 8,  # 中段起始层（含），0-index
43                mlp_back_start: int | None = 16,  # 后段起始层（含），0-index
44                mlp_ratio_overrides: dict | None = {21: 5.0,22: 5.0, 23: 5.0},
45                mlp_ratio=4.5,
46                ignore_index=-100,
47                loss_reduction: str = "token_mean",
48                sample_mean_alpha: float = 0.75,
49                drop_path=0.00,
50                residual_dropout=0.00,
51                moe_layers=[],
52                num_expert=4,
53                moe_use_detach=False,
54                moe_cap_factor=1.5,
55                moe_aux_weight=5.0,
56                use_checkpoint=False,
57                checkpoint_use_reentrant=False,
58                use_adaptive_softmax: bool = True,
59                adaptive_cutoffs = [20000, 60000],
60                 vision_dropout=0.0,
61                adaptive_div: float = 4.0,
62                adaptive_calibrate_every: int = 200,
63                 use_vision: bool = False,
64                 vision_feature_dim: int | None =1024,  # ViT 输出维度，比如 1024/768
65                 max_vision_tokens: int = 0,  # 视觉 token 上限
66                 vision_mlp_ratio: float = 4.0,
67                 vision_use_swiglu: bool = True,
68
69                **kwargs):
70        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id,pad_token_id=pad_token_id,**kwargs)
71        self.vocab_size = vocab_size
72        self.hidden_size = hidden_size
73        self.num_hidden_layers = num_hidden_layers         #transformer层数
74        self.num_attention_heads = num_attention_heads     #q head数
75        self.num_key_value_heads = num_key_value_heads     #kv head数，q head % k head=0
76        self.max_position_embeddings = max_position_embeddings # 理论上下文极限值，已达到tokenizer 上限，扩展需修改tokenizer
77        self.RoPE_base=RoPE_base                           # RoPE时频率相关theta base大小，越大支持上下文越多，目前支持最大4k，1e6可支持16k，但针对短文本敏感降低
78        self.rope_type=rope_type                        #RoPE时是否开启动态NTK扩展当前base 4k的能力到16k，一般训练可关闭
79        self.dropout = dropout                             #控制attention和MLP dropout概率
80        self.train_maxlength=train_maxlength               #训练时文本最大长度4096
81        self.rms_norm_eps = rms_norm_eps                   #非attention内部rms norm时计算rms时算完平均值后在rsqrt前增加的值
82        self.qkrms_norm_eps = qkrms_norm_eps               # attention内部在qk投影后rms norm时计算rms时算完平均值后在rsqrt前增加的值
83        self.use_qk_norm = use_qk_norm                      #是否启用qk norm
84        self.kv_cache_dtype = kv_cache_dtype
85        self.use_moe = use_moe
86        self.adapter_type = adapter_type
87        self.bos_token_id = bos_token_id
88        self.eos_token_id = eos_token_id
89        self.use_ssm=use_ssm
90        self.ssm_layers=ssm_layers
91        self.embeddingdropout=embeddingdropout          #embedding dropout概率，默认0，最大0.05
92        self.learnable_temp= learnable_temp             #是否对qkrms_norm启用一个可学习的增幅
93        self.use_affine=use_affine                      #是否对rms_norm启用一个可学习的增幅
94        self.use_sampleatt=use_sampleattion             #是否使用自己写的attention，效率低，无cuda优化
95        self.use_swiGLU=use_swiGLU                      #MLP是否使用swiGLU，False使用up-silu-down
96        self.mlp_ratio=mlp_ratio                        #MLP参数量2*mlp_ratio*H^2
97        self.ignore_index=ignore_index
98        self.loss_reduction = str(loss_reduction)
99        self.sample_mean_alpha = float(sample_mean_alpha)
100        if self.loss_reduction not in {"token_mean", "sample_mean", "hybrid"}:
101            raise ValueError(
102                "loss_reduction must be one of: token_mean, sample_mean, hybrid"
103            )
104        if not 0.0 <= self.sample_mean_alpha <= 1.0:
105            raise ValueError("sample_mean_alpha must be within [0, 1]")
106        self.drop_path=drop_path                        #attention MLP 残差分支最大droppath的概率,随着层数线性增长至最大值
107        self.residual_dropout=residual_dropout          #attention MLP 残差分支逐元素drop的概率
108        self.use_moe=use_moe
109        self.moe_layers=moe_layers                      #moe加载在哪些层
110        self.num_expert=num_expert
111        self.moe_use_detach=moe_use_detach              # false让主损失的梯度回到moe gate，训练不稳但能真的帮助gate学会使用不同的expert，否则gate趋于平均分布
112        self.moe_cap_factor=moe_cap_factor              #1-1.5之间选择，决定每个expert加载的最大token量
113        self.moe_aux_weight=moe_aux_weight
114        self.use_checkpoint = use_checkpoint                    #开启训练慢降显存
115        self.checkpoint_use_reentrant = checkpoint_use_reentrant
116        self.use_adaptive_softmax=use_adaptive_softmax
117        self.adaptive_cutoffs= adaptive_cutoffs
118        self.adaptive_div=adaptive_div
119        self.adaptive_calibrate_every=adaptive_calibrate_every
120        self.use_tcn=use_tcn
121        self.tcn_layers=tcn_layers
122        self.mlp_ratio = mlp_ratio
123        
124        self.drop_high_loss=drop_high_loss
125        self.mlp_ratio_front = float(mlp_ratio_front)
126        self.mlp_ratio_mid   = float(mlp_ratio_mid)
127        self.mlp_ratio_back  = float(mlp_ratio_back)
128        self.mlp_mid_start   = mlp_mid_start
129        self.mlp_back_start  = mlp_back_start
130        self.mlp_ratio_overrides = mlp_ratio_overrides or {}
131        self.eso_loss_radio=eso_loss_radio
132        self.use_vision = use_vision
133        self.vision_feature_dim = vision_feature_dim
134        self.max_vision_tokens = max_vision_tokens
135        self.vision_mlp_ratio = vision_mlp_ratio
136        self.vision_use_swiglu = vision_use_swiglu
137        self.vision_dropout = vision_dropout if vision_dropout is not None else dropout
138        self.num_query_tokens=num_query_tokens
139