Modelpublic

Data-Gouv-ML/RNE

sourceHugging Faceapache-2.0updated 3mo agoView on Hugging Face

0likes6downloads

config.json29 linesDownload Raw Back to root

1{2  "model_type": "rne_tiny_gpt",3  "architectures": [4    "RNETinyGPTModel"5  ],6  "auto_map": {7    "AutoConfig": "configuration_rne_tiny_gpt.RNETinyGPTConfig",8    "AutoModel": "modeling_rne_tiny_gpt.RNETinyGPTModel"9  },10  "vocab_size": 32768,11  "ctx_len": 3072,12  "max_position_embeddings": 3072,13  "n_layer": 4,14  "num_hidden_layers": 4,15  "n_head": 4,16  "num_attention_heads": 4,17  "n_embd": 384,18  "hidden_size": 384,19  "dropout": 0.0,20  "pad_token_id": 0,21  "sep_token_id": 3,22  "pooling": "mean",23  "normalize_embeddings": true,24  "attention_backend": "torch",25  "torch_fallback": false,26  "torch_dtype": "bfloat16",27  "transformers_version": "custom"28}29