QuixiAI/FlyGPT

1"""FlyGPT configuration for Hugging Face `transformers` (loaded with trust_remote_code=True)."""2from transformers import PretrainedConfig3 4 5class FlyGPTConfig(PretrainedConfig):6    model_type = "flygpt"7 8    def __init__(9        self,10        vocab_size: int = 65,11        num_neurons: int = 0,12        num_edges: int = 0,13        embedding_dim: int = 32,14        num_input_neurons: int = 256,15        num_output_neurons: int = 512,16        microsteps: int = 2,17        activation: str = "tanh",18        learned_leak: bool = True,19        leak_init: float = 0.5,20        degree_normalization: bool = True,21        init_scale: float = 0.1,22        graph: dict | None = None,23        training_state: dict | None = None,24        **kwargs,25    ):26        self.vocab_size = vocab_size27        self.num_neurons = num_neurons28        self.num_edges = num_edges29        self.embedding_dim = embedding_dim30        self.num_input_neurons = num_input_neurons31        self.num_output_neurons = num_output_neurons32        self.microsteps = microsteps33        self.activation = activation34        self.learned_leak = learned_leak35        self.leak_init = leak_init36        self.degree_normalization = degree_normalization37        self.init_scale = init_scale38        self.graph = graph or {}                    # source, hash, region filter, stats, provenance39        self.training_state = training_state or {}  # "init" or the run that produced the values40        super().__init__(**kwargs)41