connaaa/interpgpt-adhd-23M

1"""2HuggingFace PretrainedConfig for InterpGPT / TaskGPT.3 4Mirrors gpt_model.GPTConfig but subclasses transformers.PretrainedConfig5so `AutoConfig` / `AutoModel.from_pretrained(..., trust_remote_code=True)` work.6"""7from transformers import PretrainedConfig8 9 10class InterpGPTConfig(PretrainedConfig):11    model_type = "interpgpt"12 13    def __init__(14        self,15        vocab_size: int = 8192,16        max_seq_len: int = 512,17        n_layers: int = 6,18        n_heads: int = 8,19        d_model: int = 512,20        d_ff: int = 2048,21        dropout: float = 0.1,22        pad_id: int = 0,23        bias: bool = False,24        variant: str = "standard",25        **kwargs,26    ):27        self.vocab_size = vocab_size28        self.max_seq_len = max_seq_len29        self.n_layers = n_layers30        self.n_heads = n_heads31        self.d_model = d_model32        self.d_ff = d_ff33        self.dropout = dropout34        self.pad_id = pad_id35        self.bias = bias36        self.variant = variant37        kwargs.pop("pad_token_id", None)38        super().__init__(pad_token_id=pad_id, **kwargs)39