trl-internal-testing/tiny-Cohere2ForCausalLM

1{2  "_sliding_window_pattern": 4,3  "architectures": [4    "Cohere2ForCausalLM"5  ],6  "attention_bias": false,7  "attention_dropout": 0.0,8  "bos_token_id": 2,9  "cache_implementation": "hybrid",10  "dtype": "bfloat16",11  "eos_token_id": 3,12  "head_dim": 2,13  "hidden_act": "silu",14  "hidden_size": 8,15  "initializer_range": 0.02,16  "intermediate_size": 32,17  "layer_norm_eps": 1e-05,18  "layer_switch": 4,19  "layer_types": [20    "sliding_attention",21    "full_attention"22  ],23  "logit_scale": 1.0,24  "max_position_embeddings": 8192,25  "model_type": "cohere2",26  "num_attention_heads": 4,27  "num_hidden_layers": 2,28  "num_key_value_heads": 2,29  "order_of_interleaved_layers": "local_attn_first",30  "pad_token_id": 0,31  "position_embedding_type": "rope_gptj",32  "rope_scaling": null,33  "rope_theta": 50000,34  "rotary_pct": 1.0,35  "sliding_window": 4096,36  "transformers_version": "4.56.2",37  "use_cache": true,38  "use_embedding_sharing": true,39  "use_gated_activation": true,40  "use_parallel_block": true,41  "use_parallel_embedding": false,42  "use_qk_norm": false,43  "vocab_size": 26214444}45