mlx-community/North-Mini-Code-1.0-4bit

1{2    "architectures": [3        "Cohere2MoeForCausalLM"4    ],5    "attention_bias": false,6    "attention_dropout": 0.0,7    "bos_token_id": 2,8    "dtype": "bfloat16",9    "eos_token_id": 255001,10    "expert_selection_fn": "sigmoid",11    "first_k_dense_replace": 1,12    "head_dim": 128,13    "hidden_act": "silu",14    "hidden_size": 2048,15    "initializer_range": 0.02,16    "intermediate_size": 768,17    "layer_norm_eps": 1e-05,18    "layer_types": [19        "full_attention",20        "sliding_attention",21        "sliding_attention",22        "sliding_attention",23        "full_attention",24        "sliding_attention",25        "sliding_attention",26        "sliding_attention",27        "full_attention",28        "sliding_attention",29        "sliding_attention",30        "sliding_attention",31        "full_attention",32        "sliding_attention",33        "sliding_attention",34        "sliding_attention",35        "full_attention",36        "sliding_attention",37        "sliding_attention",38        "sliding_attention",39        "full_attention",40        "sliding_attention",41        "sliding_attention",42        "sliding_attention",43        "full_attention",44        "sliding_attention",45        "sliding_attention",46        "sliding_attention",47        "full_attention",48        "sliding_attention",49        "sliding_attention",50        "sliding_attention",51        "full_attention",52        "sliding_attention",53        "sliding_attention",54        "sliding_attention",55        "full_attention",56        "sliding_attention",57        "sliding_attention",58        "sliding_attention",59        "full_attention",60        "sliding_attention",61        "sliding_attention",62        "sliding_attention",63        "full_attention",64        "sliding_attention",65        "sliding_attention",66        "sliding_attention",67        "full_attention"68    ],69    "logit_scale": 1.0,70    "max_position_embeddings": 500000,71    "model_type": "cohere2_moe",72    "norm_topk_prob": false,73    "num_attention_heads": 32,74    "num_experts": 128,75    "num_experts_per_tok": 8,76    "num_hidden_layers": 49,77    "num_key_value_heads": 4,78    "num_shared_experts": 0,79    "pad_token_id": 0,80    "prefix_dense_intermediate_size": 3072,81    "prefix_dense_sliding_window_pattern": 1,82    "quantization": {83        "group_size": 64,84        "bits": 4,85        "mode": "affine"86    },87    "quantization_config": {88        "group_size": 64,89        "bits": 4,90        "mode": "affine"91    },92    "rms_norm_eps": 1e-06,93    "rope_scaling": null,94    "rope_theta": 50000,95    "shared_expert_combination_strategy": "average",96    "sliding_window": 4096,97    "transformers_version": "5.8.0",98    "use_cache": true,99    "use_gated_activation": true,100    "use_parallel_block": true,101    "use_parallel_embedding": false,102    "use_qk_norm": false,103    "vision_config": {},104    "vocab_size": 262144105}