crosslingual-em/tiny-aya-fire-em-code-en-code-insecure-seed_2
018
1{2 "_sliding_window_pattern": 4,3 "architectures": [4 "Cohere2ForCausalLM"5 ],6 "attention_bias": false,7 "attention_dropout": 0.0,8 "bos_token_id": 2,9 "cache_implementation": "hybrid",10 "dtype": "bfloat16",11 "eos_token_id": 6,12 "head_dim": 128,13 "hidden_act": "silu",14 "hidden_size": 2048,15 "initializer_range": 0.02,16 "intermediate_size": 11008,17 "layer_norm_eps": 1e-05,18 "layer_switch": 4,19 "layer_types": [20 "sliding_attention",21 "sliding_attention",22 "sliding_attention",23 "full_attention",24 "sliding_attention",25 "sliding_attention",26 "sliding_attention",27 "full_attention",28 "sliding_attention",29 "sliding_attention",30 "sliding_attention",31 "full_attention",32 "sliding_attention",33 "sliding_attention",34 "sliding_attention",35 "full_attention",36 "sliding_attention",37 "sliding_attention",38 "sliding_attention",39 "full_attention",40 "sliding_attention",41 "sliding_attention",42 "sliding_attention",43 "full_attention",44 "sliding_attention",45 "sliding_attention",46 "sliding_attention",47 "full_attention",48 "sliding_attention",49 "sliding_attention",50 "sliding_attention",51 "full_attention",52 "sliding_attention",53 "sliding_attention",54 "sliding_attention",55 "full_attention"56 ],57 "logit_scale": 1.0,58 "max_position_embeddings": 500000,59 "model_type": "cohere2",60 "num_attention_heads": 16,61 "num_hidden_layers": 36,62 "num_key_value_heads": 4,63 "order_of_interleaved_layers": "local_attn_first",64 "pad_token_id": 0,65 "position_embedding_type": "rope_gptj",66 "rope_parameters": {67 "rope_theta": 50000,68 "rope_type": "default"69 },70 "rotary_pct": 1.0,71 "sliding_window": 4096,72 "sliding_window_pattern": 4,73 "tie_word_embeddings": true,74 "transformers_version": "5.0.0",75 "use_cache": false,76 "use_embedding_sharing": true,77 "use_gated_activation": true,78 "use_parallel_block": true,79 "use_parallel_embedding": false,80 "use_qk_norm": false,81 "vocab_size": 26214482}83