txgsync/Maple-Preview-BF16-MLX

1{2  "architectures": [3    "MapleForCausalLM"4  ],5  "attention_dropout": 0.0,6  "bos_token_id": 151643,7  "dtype": "bfloat16",8  "embedding_dropout": 0.0,9  "eos_token_id": 151645,10  "head_dim": 128,11  "hidden_act": "silu",12  "hidden_size": 2048,13  "initializer_range": 0.02,14  "intermediate_size": 4096,15  "layer_types": [16    "sliding_attention",17    "sliding_attention",18    "sliding_attention",19    "full_attention",20    "sliding_attention",21    "sliding_attention",22    "sliding_attention",23    "full_attention",24    "sliding_attention",25    "sliding_attention",26    "sliding_attention",27    "full_attention",28    "sliding_attention",29    "sliding_attention",30    "sliding_attention",31    "full_attention",32    "sliding_attention",33    "sliding_attention",34    "sliding_attention",35    "full_attention",36    "sliding_attention",37    "sliding_attention",38    "sliding_attention",39    "full_attention"40  ],41  "max_position_embeddings": 131072,42  "max_window_layers": 24,43  "model_file": "maple.py",44  "model_type": "maple",45  "moe_intermediate_size": 512,46  "moe_router_enable_expert_bias": false,47  "nope_on_global_attention": true,48  "norm_topk_prob": true,49  "num_attention_heads": 16,50  "num_experts": 256,51  "num_experts_per_tok": 8,52  "num_hidden_layers": 24,53  "num_key_value_heads": 4,54  "num_shared_experts": 0,55  "output_dropout": 0.0,56  "output_router_logits": false,57  "pad_token_id": null,58  "partial_rotary_factor": 0.5,59  "preaffine": false,60  "rms_norm_eps": 1e-06,61  "rope_scaling": null,62  "rope_theta": 10000,63  "router_dtype": "fp32",64  "sliding_window": 512,65  "tie_word_embeddings": false,66  "transformers_version": "4.57.1",67  "use_cache": true,68  "use_qk_norm": true,69  "use_rmsnorm": true,70  "vocab_size": 15193671}72