ShaunGves/North-Mini-Code-1.0

1{2  "architectures": [3    "Cohere2MoeForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "bos_token_id": 2,8  "dtype": "bfloat16",9  "eos_token_id": 255001,10  "expert_selection_fn": "sigmoid",11  "first_k_dense_replace": 1,12  "head_dim": 128,13  "hidden_act": "silu",14  "hidden_size": 2048,15  "initializer_range": 0.02,16  "intermediate_size": 768,17  "layer_norm_eps": 1e-05,18  "layer_types": [19    "full_attention",20    "sliding_attention",21    "sliding_attention",22    "sliding_attention",23    "full_attention",24    "sliding_attention",25    "sliding_attention",26    "sliding_attention",27    "full_attention",28    "sliding_attention",29    "sliding_attention",30    "sliding_attention",31    "full_attention",32    "sliding_attention",33    "sliding_attention",34    "sliding_attention",35    "full_attention",36    "sliding_attention",37    "sliding_attention",38    "sliding_attention",39    "full_attention",40    "sliding_attention",41    "sliding_attention",42    "sliding_attention",43    "full_attention",44    "sliding_attention",45    "sliding_attention",46    "sliding_attention",47    "full_attention",48    "sliding_attention",49    "sliding_attention",50    "sliding_attention",51    "full_attention",52    "sliding_attention",53    "sliding_attention",54    "sliding_attention",55    "full_attention",56    "sliding_attention",57    "sliding_attention",58    "sliding_attention",59    "full_attention",60    "sliding_attention",61    "sliding_attention",62    "sliding_attention",63    "full_attention",64    "sliding_attention",65    "sliding_attention",66    "sliding_attention",67    "full_attention"68  ],69  "logit_scale": 1.0,70  "max_position_embeddings": 500000,71  "model_type": "cohere2_moe",72  "norm_topk_prob": false,73  "num_attention_heads": 32,74  "num_experts": 128,75  "num_experts_per_tok": 8,76  "num_hidden_layers": 49,77  "num_key_value_heads": 4,78  "num_shared_experts": 0,79  "pad_token_id": 0,80  "prefix_dense_intermediate_size": 3072,81  "prefix_dense_sliding_window_pattern": 1,82  "rms_norm_eps": 1e-06,83  "rope_scaling": null,84  "rope_theta": 50000,85  "shared_expert_combination_strategy": "average",86  "sliding_window": 4096,87  "transformers_version": "5.8.0",88  "use_cache": true,89  "use_gated_activation": true,90  "use_parallel_block": true,91  "use_parallel_embedding": false,92  "use_qk_norm": false,93  "vocab_size": 26214494}95