mlx-community/tiny-aya-fire

1{2    "_sliding_window_pattern": 4,3    "architectures": [4        "Cohere2ForCausalLM"5    ],6    "attention_bias": false,7    "attention_dropout": 0.0,8    "bos_token_id": 2,9    "cache_implementation": "hybrid",10    "eos_token_id": 3,11    "head_dim": 128,12    "hidden_act": "silu",13    "hidden_size": 2048,14    "initializer_range": 0.02,15    "intermediate_size": 11008,16    "layer_norm_eps": 1e-05,17    "layer_switch": 4,18    "layer_types": [19        "sliding_attention",20        "sliding_attention",21        "sliding_attention",22        "full_attention",23        "sliding_attention",24        "sliding_attention",25        "sliding_attention",26        "full_attention",27        "sliding_attention",28        "sliding_attention",29        "sliding_attention",30        "full_attention",31        "sliding_attention",32        "sliding_attention",33        "sliding_attention",34        "full_attention",35        "sliding_attention",36        "sliding_attention",37        "sliding_attention",38        "full_attention",39        "sliding_attention",40        "sliding_attention",41        "sliding_attention",42        "full_attention",43        "sliding_attention",44        "sliding_attention",45        "sliding_attention",46        "full_attention",47        "sliding_attention",48        "sliding_attention",49        "sliding_attention",50        "full_attention",51        "sliding_attention",52        "sliding_attention",53        "sliding_attention",54        "full_attention"55    ],56    "logit_scale": 1.0,57    "max_position_embeddings": 500000,58    "model_type": "cohere2",59    "num_attention_heads": 16,60    "num_hidden_layers": 36,61    "num_key_value_heads": 4,62    "order_of_interleaved_layers": "local_attn_first",63    "pad_token_id": 0,64    "position_embedding_type": "rope_gptj",65    "rope_scaling": null,66    "rope_theta": 50000,67    "rotary_pct": 1.0,68    "sliding_window": 4096,69    "sliding_window_pattern": 4,70    "torch_dtype": "bfloat16",71    "transformers_version": "4.51.3",72    "use_cache": true,73    "use_embedding_sharing": true,74    "use_gated_activation": true,75    "use_parallel_block": true,76    "use_parallel_embedding": false,77    "use_qk_norm": false,78    "vocab_size": 26214479}