hf-tiny-v2/tiny-random-ModernBertDecoderModel

1{2  "architectures": [3    "ModernBertDecoderModel"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "bos_token_id": 50281,8  "classifier_activation": "gelu",9  "classifier_bias": false,10  "classifier_dropout": 0.0,11  "cls_token_id": 50281,12  "decoder_bias": true,13  "dtype": "float32",14  "embedding_dropout": 0.0,15  "eos_token_id": 50282,16  "hidden_activation": "gelu",17  "hidden_size": 32,18  "initializer_cutoff_factor": 2.0,19  "initializer_range": 0.02,20  "intermediate_size": 32,21  "layer_types": [22    "full_attention",23    "sliding_attention"24  ],25  "local_attention": 128,26  "max_position_embeddings": 512,27  "mlp_bias": false,28  "mlp_dropout": 0.0,29  "model_type": "modernbert-decoder",30  "norm_bias": false,31  "norm_eps": 1e-05,32  "num_attention_heads": 2,33  "num_hidden_layers": 2,34  "pad_token_id": 50283,35  "rope_parameters": {36    "full_attention": {37      "rope_theta": 160000.0,38      "rope_type": "default"39    },40    "sliding_attention": {41      "rope_theta": 10000.0,42      "rope_type": "default"43    }44  },45  "sep_token_id": 50282,46  "sliding_window": 64,47  "tie_word_embeddings": false,48  "transformers_version": "5.16.0.dev0",49  "use_cache": true,50  "vocab_size": 5036851}52