daslab-testing/CloverLM

1from transformers import PretrainedConfig2 3 4class CloverLMConfig(PretrainedConfig):5    model_type = "cloverlm"6 7    def __init__(8        self,9        vocab_size=32000,10        num_blocks=4,11        heads=6,12        d_head=128,13        ratio=3,14        scale_type="1/sqrt(d)",15        max_context=1024,16        quartet_2_impl="pseudoquant",17        weight_tying=True,18        attn_backend="pytorch",19        # Optional: HuggingFace / vLLM tooling (defaults derived from shape)20        hidden_size=None,21        intermediate_size=None,22        max_position_embeddings=None,23        num_attention_heads=None,24        num_key_value_heads=None,25        head_dim=None,26        **kwargs,27    ):28        self.num_blocks = num_blocks29        self.num_hidden_layers = num_blocks30        self.heads = heads31        self.d_head = d_head32        self.ratio = ratio33        self.scale_type = scale_type34        self.max_context = max_context35        self.quartet_2_impl = quartet_2_impl36        self.weight_tying = weight_tying37        self.attn_backend = attn_backend38 39        d_model = heads * d_head40        self.hidden_size = hidden_size if hidden_size is not None else d_model41        self.intermediate_size = (42            intermediate_size if intermediate_size is not None else 4 * d_model43        )44        self.max_position_embeddings = (45            max_position_embeddings46            if max_position_embeddings is not None47            else max_context48        )49        self.num_attention_heads = (50            num_attention_heads if num_attention_heads is not None else heads51        )52        self.num_key_value_heads = (53            num_key_value_heads54            if num_key_value_heads is not None55            else heads // ratio56        )57        self.head_dim = head_dim if head_dim is not None else d_head58 59        kwargs.pop("tie_word_embeddings", None)60        super().__init__(61            vocab_size=vocab_size,62            tie_word_embeddings=weight_tying,63            **kwargs,64        )65