zai-org/codegeex4-all-9b

1from transformers import PretrainedConfig2 3 4class ChatGLMConfig(PretrainedConfig):5    model_type = "chatglm"6 7    def __init__(8            self,9            num_layers=28,10            padded_vocab_size=65024,11            hidden_size=4096,12            ffn_hidden_size=13696,13            kv_channels=128,14            num_attention_heads=32,15            seq_length=2048,16            hidden_dropout=0.0,17            classifier_dropout=None,18            attention_dropout=0.0,19            layernorm_epsilon=1e-5,20            rmsnorm=True,21            apply_residual_connection_post_layernorm=False,22            post_layer_norm=True,23            add_bias_linear=False,24            add_qkv_bias=False,25            bias_dropout_fusion=True,26            multi_query_attention=False,27            multi_query_group_num=1,28            rope_ratio=1,29            apply_query_key_layer_scaling=True,30            attention_softmax_in_fp32=True,31            fp32_residual_connection=False,32            **kwargs33    ):34        self.num_layers = num_layers35        self.vocab_size = padded_vocab_size36        self.padded_vocab_size = padded_vocab_size37        self.hidden_size = hidden_size38        self.ffn_hidden_size = ffn_hidden_size39        self.kv_channels = kv_channels40        self.num_attention_heads = num_attention_heads41        self.seq_length = seq_length42        self.hidden_dropout = hidden_dropout43        self.classifier_dropout = classifier_dropout44        self.attention_dropout = attention_dropout45        self.layernorm_epsilon = layernorm_epsilon46        self.rmsnorm = rmsnorm47        self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm48        self.post_layer_norm = post_layer_norm49        self.add_bias_linear = add_bias_linear50        self.add_qkv_bias = add_qkv_bias51        self.bias_dropout_fusion = bias_dropout_fusion52        self.multi_query_attention = multi_query_attention53        self.multi_query_group_num = multi_query_group_num54        self.rope_ratio = rope_ratio55        self.apply_query_key_layer_scaling = apply_query_key_layer_scaling56        self.attention_softmax_in_fp32 = attention_softmax_in_fp3257        self.fp32_residual_connection = fp32_residual_connection58        super().__init__(**kwargs)59