mlx-community/North-Mini-Code-1.0-bf16
179
1{2 "architectures": [3 "Cohere2MoeForCausalLM"4 ],5 "attention_bias": false,6 "attention_dropout": 0.0,7 "bos_token_id": 2,8 "dtype": "bfloat16",9 "eos_token_id": 255001,10 "expert_selection_fn": "sigmoid",11 "first_k_dense_replace": 1,12 "head_dim": 128,13 "hidden_act": "silu",14 "hidden_size": 2048,15 "initializer_range": 0.02,16 "intermediate_size": 768,17 "layer_norm_eps": 1e-05,18 "layer_types": [19 "full_attention",20 "sliding_attention",21 "sliding_attention",22 "sliding_attention",23 "full_attention",24 "sliding_attention",25 "sliding_attention",26 "sliding_attention",27 "full_attention",28 "sliding_attention",29 "sliding_attention",30 "sliding_attention",31 "full_attention",32 "sliding_attention",33 "sliding_attention",34 "sliding_attention",35 "full_attention",36 "sliding_attention",37 "sliding_attention",38 "sliding_attention",39 "full_attention",40 "sliding_attention",41 "sliding_attention",42 "sliding_attention",43 "full_attention",44 "sliding_attention",45 "sliding_attention",46 "sliding_attention",47 "full_attention",48 "sliding_attention",49 "sliding_attention",50 "sliding_attention",51 "full_attention",52 "sliding_attention",53 "sliding_attention",54 "sliding_attention",55 "full_attention",56 "sliding_attention",57 "sliding_attention",58 "sliding_attention",59 "full_attention",60 "sliding_attention",61 "sliding_attention",62 "sliding_attention",63 "full_attention",64 "sliding_attention",65 "sliding_attention",66 "sliding_attention",67 "full_attention"68 ],69 "logit_scale": 1.0,70 "max_position_embeddings": 500000,71 "model_type": "cohere2_moe",72 "norm_topk_prob": false,73 "num_attention_heads": 32,74 "num_experts": 128,75 "num_experts_per_tok": 8,76 "num_hidden_layers": 49,77 "num_key_value_heads": 4,78 "num_shared_experts": 0,79 "pad_token_id": 0,80 "prefix_dense_intermediate_size": 3072,81 "prefix_dense_sliding_window_pattern": 1,82 "rms_norm_eps": 1e-06,83 "rope_scaling": null,84 "rope_theta": 50000,85 "shared_expert_combination_strategy": "average",86 "sliding_window": 4096,87 "transformers_version": "5.8.0",88 "use_cache": true,89 "use_gated_activation": true,90 "use_parallel_block": true,91 "use_parallel_embedding": false,92 "use_qk_norm": false,93 "vocab_size": 26214494}