majentik/Ornith-1.5-35B-A3B-MLX-3bit
0208
1{2 "architectures": [3 "Qwen3_5MoeForConditionalGeneration"4 ],5 "bos_token_id": null,6 "do_sample": true,7 "dtype": "bfloat16",8 "eos_token_id": [9 248046,10 24804411 ],12 "generation_config": {13 "bos_token_id": 248044,14 "do_sample": true,15 "eos_token_id": [16 248046,17 24804418 ],19 "pad_token_id": 248044,20 "temperature": 1.0,21 "top_k": 20,22 "top_p": 0.9523 },24 "hidden_size": 2048,25 "image_token_id": 248056,26 "model_type": "qwen3_5_moe",27 "pad_token_id": 248044,28 "quantization": {29 "group_size": 32,30 "bits": 3,31 "mode": "affine",32 "language_model.model.layers.0.mlp.gate": {33 "group_size": 64,34 "bits": 835 },36 "language_model.model.layers.0.mlp.shared_expert_gate": {37 "group_size": 64,38 "bits": 839 },40 "language_model.model.layers.1.mlp.gate": {41 "group_size": 64,42 "bits": 843 },44 "language_model.model.layers.1.mlp.shared_expert_gate": {45 "group_size": 64,46 "bits": 847 },48 "language_model.model.layers.2.mlp.gate": {49 "group_size": 64,50 "bits": 851 },52 "language_model.model.layers.2.mlp.shared_expert_gate": {53 "group_size": 64,54 "bits": 855 },56 "language_model.model.layers.3.mlp.gate": {57 "group_size": 64,58 "bits": 859 },60 "language_model.model.layers.3.mlp.shared_expert_gate": {61 "group_size": 64,62 "bits": 863 },64 "language_model.model.layers.4.mlp.gate": {65 "group_size": 64,66 "bits": 867 },68 "language_model.model.layers.4.mlp.shared_expert_gate": {69 "group_size": 64,70 "bits": 871 },72 "language_model.model.layers.5.mlp.gate": {73 "group_size": 64,74 "bits": 875 },76 "language_model.model.layers.5.mlp.shared_expert_gate": {77 "group_size": 64,78 "bits": 879 },80 "language_model.model.layers.6.mlp.gate": {81 "group_size": 64,82 "bits": 883 },84 "language_model.model.layers.6.mlp.shared_expert_gate": {85 "group_size": 64,86 "bits": 887 },88 "language_model.model.layers.7.mlp.gate": {89 "group_size": 64,90 "bits": 891 },92 "language_model.model.layers.7.mlp.shared_expert_gate": {93 "group_size": 64,94 "bits": 895 },96 "language_model.model.layers.8.mlp.gate": {97 "group_size": 64,98 "bits": 899 },100 "language_model.model.layers.8.mlp.shared_expert_gate": {101 "group_size": 64,102 "bits": 8103 },104 "language_model.model.layers.9.mlp.gate": {105 "group_size": 64,106 "bits": 8107 },108 "language_model.model.layers.9.mlp.shared_expert_gate": {109 "group_size": 64,110 "bits": 8111 },112 "language_model.model.layers.10.mlp.gate": {113 "group_size": 64,114 "bits": 8115 },116 "language_model.model.layers.10.mlp.shared_expert_gate": {117 "group_size": 64,118 "bits": 8119 },120 "language_model.model.layers.11.mlp.gate": {121 "group_size": 64,122 "bits": 8123 },124 "language_model.model.layers.11.mlp.shared_expert_gate": {125 "group_size": 64,126 "bits": 8127 },128 "language_model.model.layers.12.mlp.gate": {129 "group_size": 64,130 "bits": 8131 },132 "language_model.model.layers.12.mlp.shared_expert_gate": {133 "group_size": 64,134 "bits": 8135 },136 "language_model.model.layers.13.mlp.gate": {137 "group_size": 64,138 "bits": 8139 },140 "language_model.model.layers.13.mlp.shared_expert_gate": {141 "group_size": 64,142 "bits": 8143 },144 "language_model.model.layers.14.mlp.gate": {145 "group_size": 64,146 "bits": 8147 },148 "language_model.model.layers.14.mlp.shared_expert_gate": {149 "group_size": 64,150 "bits": 8151 },152 "language_model.model.layers.15.mlp.gate": {153 "group_size": 64,154 "bits": 8155 },156 "language_model.model.layers.15.mlp.shared_expert_gate": {157 "group_size": 64,158 "bits": 8159 },160 "language_model.model.layers.16.mlp.gate": {161 "group_size": 64,162 "bits": 8163 },164 "language_model.model.layers.16.mlp.shared_expert_gate": {165 "group_size": 64,166 "bits": 8167 },168 "language_model.model.layers.17.mlp.gate": {169 "group_size": 64,170 "bits": 8171 },172 "language_model.model.layers.17.mlp.shared_expert_gate": {173 "group_size": 64,174 "bits": 8175 },176 "language_model.model.layers.18.mlp.gate": {177 "group_size": 64,178 "bits": 8179 },180 "language_model.model.layers.18.mlp.shared_expert_gate": {181 "group_size": 64,182 "bits": 8183 },184 "language_model.model.layers.19.mlp.gate": {185 "group_size": 64,186 "bits": 8187 },188 "language_model.model.layers.19.mlp.shared_expert_gate": {189 "group_size": 64,190 "bits": 8191 },192 "language_model.model.layers.20.mlp.gate": {193 "group_size": 64,194 "bits": 8195 },196 "language_model.model.layers.20.mlp.shared_expert_gate": {197 "group_size": 64,198 "bits": 8199 },200 "language_model.model.layers.21.mlp.gate": {201 "group_size": 64,202 "bits": 8203 },204 "language_model.model.layers.21.mlp.shared_expert_gate": {205 "group_size": 64,206 "bits": 8207 },208 "language_model.model.layers.22.mlp.gate": {209 "group_size": 64,210 "bits": 8211 },212 "language_model.model.layers.22.mlp.shared_expert_gate": {213 "group_size": 64,214 "bits": 8215 },216 "language_model.model.layers.23.mlp.gate": {217 "group_size": 64,218 "bits": 8219 },220 "language_model.model.layers.23.mlp.shared_expert_gate": {221 "group_size": 64,222 "bits": 8223 },224 "language_model.model.layers.24.mlp.gate": {225 "group_size": 64,226 "bits": 8227 },228 "language_model.model.layers.24.mlp.shared_expert_gate": {229 "group_size": 64,230 "bits": 8231 },232 "language_model.model.layers.25.mlp.gate": {233 "group_size": 64,234 "bits": 8235 },236 "language_model.model.layers.25.mlp.shared_expert_gate": {237 "group_size": 64,238 "bits": 8239 },240 "language_model.model.layers.26.mlp.gate": {241 "group_size": 64,242 "bits": 8243 },244 "language_model.model.layers.26.mlp.shared_expert_gate": {245 "group_size": 64,246 "bits": 8247 },248 "language_model.model.layers.27.mlp.gate": {249 "group_size": 64,250 "bits": 8251 },252 "language_model.model.layers.27.mlp.shared_expert_gate": {253 "group_size": 64,254 "bits": 8255 },256 "language_model.model.layers.28.mlp.gate": {257 "group_size": 64,258 "bits": 8259 },260 "language_model.model.layers.28.mlp.shared_expert_gate": {261 "group_size": 64,262 "bits": 8263 },264 "language_model.model.layers.29.mlp.gate": {265 "group_size": 64,266 "bits": 8267 },268 "language_model.model.layers.29.mlp.shared_expert_gate": {269 "group_size": 64,270 "bits": 8271 },272 "language_model.model.layers.30.mlp.gate": {273 "group_size": 64,274 "bits": 8275 },276 "language_model.model.layers.30.mlp.shared_expert_gate": {277 "group_size": 64,278 "bits": 8279 },280 "language_model.model.layers.31.mlp.gate": {281 "group_size": 64,282 "bits": 8283 },284 "language_model.model.layers.31.mlp.shared_expert_gate": {285 "group_size": 64,286 "bits": 8287 },288 "language_model.model.layers.32.mlp.gate": {289 "group_size": 64,290 "bits": 8291 },292 "language_model.model.layers.32.mlp.shared_expert_gate": {293 "group_size": 64,294 "bits": 8295 },296 "language_model.model.layers.33.mlp.gate": {297 "group_size": 64,298 "bits": 8299 },300 "language_model.model.layers.33.mlp.shared_expert_gate": {301 "group_size": 64,302 "bits": 8303 },304 "language_model.model.layers.34.mlp.gate": {305 "group_size": 64,306 "bits": 8307 },308 "language_model.model.layers.34.mlp.shared_expert_gate": {309 "group_size": 64,310 "bits": 8311 },312 "language_model.model.layers.35.mlp.gate": {313 "group_size": 64,314 "bits": 8315 },316 "language_model.model.layers.35.mlp.shared_expert_gate": {317 "group_size": 64,318 "bits": 8319 },320 "language_model.model.layers.36.mlp.gate": {321 "group_size": 64,322 "bits": 8323 },324 "language_model.model.layers.36.mlp.shared_expert_gate": {325 "group_size": 64,326 "bits": 8327 },328 "language_model.model.layers.37.mlp.gate": {329 "group_size": 64,330 "bits": 8331 },332 "language_model.model.layers.37.mlp.shared_expert_gate": {333 "group_size": 64,334 "bits": 8335 },336 "language_model.model.layers.38.mlp.gate": {337 "group_size": 64,338 "bits": 8339 },340 "language_model.model.layers.38.mlp.shared_expert_gate": {341 "group_size": 64,342 "bits": 8343 },344 "language_model.model.layers.39.mlp.gate": {345 "group_size": 64,346 "bits": 8347 },348 "language_model.model.layers.39.mlp.shared_expert_gate": {349 "group_size": 64,350 "bits": 8351 }352 },353 "quantization_config": {354 "group_size": 32,355 "bits": 3,356 "mode": "affine",357 "language_model.model.layers.0.mlp.gate": {358 "group_size": 64,359 "bits": 8360 },361 "language_model.model.layers.0.mlp.shared_expert_gate": {362 "group_size": 64,363 "bits": 8364 },365 "language_model.model.layers.1.mlp.gate": {366 "group_size": 64,367 "bits": 8368 },369 "language_model.model.layers.1.mlp.shared_expert_gate": {370 "group_size": 64,371 "bits": 8372 },373 "language_model.model.layers.2.mlp.gate": {374 "group_size": 64,375 "bits": 8376 },377 "language_model.model.layers.2.mlp.shared_expert_gate": {378 "group_size": 64,379 "bits": 8380 },381 "language_model.model.layers.3.mlp.gate": {382 "group_size": 64,383 "bits": 8384 },385 "language_model.model.layers.3.mlp.shared_expert_gate": {386 "group_size": 64,387 "bits": 8388 },389 "language_model.model.layers.4.mlp.gate": {390 "group_size": 64,391 "bits": 8392 },393 "language_model.model.layers.4.mlp.shared_expert_gate": {394 "group_size": 64,395 "bits": 8396 },397 "language_model.model.layers.5.mlp.gate": {398 "group_size": 64,399 "bits": 8400 },401 "language_model.model.layers.5.mlp.shared_expert_gate": {402 "group_size": 64,403 "bits": 8404 },405 "language_model.model.layers.6.mlp.gate": {406 "group_size": 64,407 "bits": 8408 },409 "language_model.model.layers.6.mlp.shared_expert_gate": {410 "group_size": 64,411 "bits": 8412 },413 "language_model.model.layers.7.mlp.gate": {414 "group_size": 64,415 "bits": 8416 },417 "language_model.model.layers.7.mlp.shared_expert_gate": {418 "group_size": 64,419 "bits": 8420 },421 "language_model.model.layers.8.mlp.gate": {422 "group_size": 64,423 "bits": 8424 },425 "language_model.model.layers.8.mlp.shared_expert_gate": {426 "group_size": 64,427 "bits": 8428 },429 "language_model.model.layers.9.mlp.gate": {430 "group_size": 64,431 "bits": 8432 },433 "language_model.model.layers.9.mlp.shared_expert_gate": {434 "group_size": 64,435 "bits": 8436 },437 "language_model.model.layers.10.mlp.gate": {438 "group_size": 64,439 "bits": 8440 },441 "language_model.model.layers.10.mlp.shared_expert_gate": {442 "group_size": 64,443 "bits": 8444 },445 "language_model.model.layers.11.mlp.gate": {446 "group_size": 64,447 "bits": 8448 },449 "language_model.model.layers.11.mlp.shared_expert_gate": {450 "group_size": 64,451 "bits": 8452 },453 "language_model.model.layers.12.mlp.gate": {454 "group_size": 64,455 "bits": 8456 },457 "language_model.model.layers.12.mlp.shared_expert_gate": {458 "group_size": 64,459 "bits": 8460 },461 "language_model.model.layers.13.mlp.gate": {462 "group_size": 64,463 "bits": 8464 },465 "language_model.model.layers.13.mlp.shared_expert_gate": {466 "group_size": 64,467 "bits": 8468 },469 "language_model.model.layers.14.mlp.gate": {470 "group_size": 64,471 "bits": 8472 },473 "language_model.model.layers.14.mlp.shared_expert_gate": {474 "group_size": 64,475 "bits": 8476 },477 "language_model.model.layers.15.mlp.gate": {478 "group_size": 64,479 "bits": 8480 },481 "language_model.model.layers.15.mlp.shared_expert_gate": {482 "group_size": 64,483 "bits": 8484 },485 "language_model.model.layers.16.mlp.gate": {486 "group_size": 64,487 "bits": 8488 },489 "language_model.model.layers.16.mlp.shared_expert_gate": {490 "group_size": 64,491 "bits": 8492 },493 "language_model.model.layers.17.mlp.gate": {494 "group_size": 64,495 "bits": 8496 },497 "language_model.model.layers.17.mlp.shared_expert_gate": {498 "group_size": 64,499 "bits": 8500 },501 "language_model.model.layers.18.mlp.gate": {502 "group_size": 64,503 "bits": 8504 },505 "language_model.model.layers.18.mlp.shared_expert_gate": {506 "group_size": 64,507 "bits": 8508 },509 "language_model.model.layers.19.mlp.gate": {510 "group_size": 64,511 "bits": 8512 },513 "language_model.model.layers.19.mlp.shared_expert_gate": {514 "group_size": 64,515 "bits": 8516 },517 "language_model.model.layers.20.mlp.gate": {518 "group_size": 64,519 "bits": 8520 },521 "language_model.model.layers.20.mlp.shared_expert_gate": {522 "group_size": 64,523 "bits": 8524 },525 "language_model.model.layers.21.mlp.gate": {526 "group_size": 64,527 "bits": 8528 },529 "language_model.model.layers.21.mlp.shared_expert_gate": {530 "group_size": 64,531 "bits": 8532 },533 "language_model.model.layers.22.mlp.gate": {534 "group_size": 64,535 "bits": 8536 },537 "language_model.model.layers.22.mlp.shared_expert_gate": {538 "group_size": 64,539 "bits": 8540 },541 "language_model.model.layers.23.mlp.gate": {542 "group_size": 64,543 "bits": 8544 },545 "language_model.model.layers.23.mlp.shared_expert_gate": {546 "group_size": 64,547 "bits": 8548 },549 "language_model.model.layers.24.mlp.gate": {550 "group_size": 64,551 "bits": 8552 },553 "language_model.model.layers.24.mlp.shared_expert_gate": {554 "group_size": 64,555 "bits": 8556 },557 "language_model.model.layers.25.mlp.gate": {558 "group_size": 64,559 "bits": 8560 },561 "language_model.model.layers.25.mlp.shared_expert_gate": {562 "group_size": 64,563 "bits": 8564 },565 "language_model.model.layers.26.mlp.gate": {566 "group_size": 64,567 "bits": 8568 },569 "language_model.model.layers.26.mlp.shared_expert_gate": {570 "group_size": 64,571 "bits": 8572 },573 "language_model.model.layers.27.mlp.gate": {574 "group_size": 64,575 "bits": 8576 },577 "language_model.model.layers.27.mlp.shared_expert_gate": {578 "group_size": 64,579 "bits": 8580 },581 "language_model.model.layers.28.mlp.gate": {582 "group_size": 64,583 "bits": 8584 },585 "language_model.model.layers.28.mlp.shared_expert_gate": {586 "group_size": 64,587 "bits": 8588 },589 "language_model.model.layers.29.mlp.gate": {590 "group_size": 64,591 "bits": 8592 },593 "language_model.model.layers.29.mlp.shared_expert_gate": {594 "group_size": 64,595 "bits": 8596 },597 "language_model.model.layers.30.mlp.gate": {598 "group_size": 64,599 "bits": 8600 },601 "language_model.model.layers.30.mlp.shared_expert_gate": {602 "group_size": 64,603 "bits": 8604 },605 "language_model.model.layers.31.mlp.gate": {606 "group_size": 64,607 "bits": 8608 },609 "language_model.model.layers.31.mlp.shared_expert_gate": {610 "group_size": 64,611 "bits": 8612 },613 "language_model.model.layers.32.mlp.gate": {614 "group_size": 64,615 "bits": 8616 },617 "language_model.model.layers.32.mlp.shared_expert_gate": {618 "group_size": 64,619 "bits": 8620 },621 "language_model.model.layers.33.mlp.gate": {622 "group_size": 64,623 "bits": 8624 },625 "language_model.model.layers.33.mlp.shared_expert_gate": {626 "group_size": 64,627 "bits": 8628 },629 "language_model.model.layers.34.mlp.gate": {630 "group_size": 64,631 "bits": 8632 },633 "language_model.model.layers.34.mlp.shared_expert_gate": {634 "group_size": 64,635 "bits": 8636 },637 "language_model.model.layers.35.mlp.gate": {638 "group_size": 64,639 "bits": 8640 },641 "language_model.model.layers.35.mlp.shared_expert_gate": {642 "group_size": 64,643 "bits": 8644 },645 "language_model.model.layers.36.mlp.gate": {646 "group_size": 64,647 "bits": 8648 },649 "language_model.model.layers.36.mlp.shared_expert_gate": {650 "group_size": 64,651 "bits": 8652 },653 "language_model.model.layers.37.mlp.gate": {654 "group_size": 64,655 "bits": 8656 },657 "language_model.model.layers.37.mlp.shared_expert_gate": {658 "group_size": 64,659 "bits": 8660 },661 "language_model.model.layers.38.mlp.gate": {662 "group_size": 64,663 "bits": 8664 },665 "language_model.model.layers.38.mlp.shared_expert_gate": {666 "group_size": 64,667 "bits": 8668 },669 "language_model.model.layers.39.mlp.gate": {670 "group_size": 64,671 "bits": 8672 },673 "language_model.model.layers.39.mlp.shared_expert_gate": {674 "group_size": 64,675 "bits": 8676 }677 },678 "temperature": 1.0,679 "text_config": {680 "attention_bias": false,681 "attention_dropout": 0.0,682 "attn_output_gate": true,683 "bos_token_id": 248044,684 "dtype": "bfloat16",685 "eos_token_id": 248044,686 "full_attention_interval": 4,687 "head_dim": 256,688 "hidden_act": "silu",689 "hidden_size": 2048,690 "initializer_range": 0.02,691 "layer_types": [692 "linear_attention",693 "linear_attention",694 "linear_attention",695 "full_attention",696 "linear_attention",697 "linear_attention",698 "linear_attention",699 "full_attention",700 "linear_attention",701 "linear_attention",702 "linear_attention",703 "full_attention",704 "linear_attention",705 "linear_attention",706 "linear_attention",707 "full_attention",708 "linear_attention",709 "linear_attention",710 "linear_attention",711 "full_attention",712 "linear_attention",713 "linear_attention",714 "linear_attention",715 "full_attention",716 "linear_attention",717 "linear_attention",718 "linear_attention",719 "full_attention",720 "linear_attention",721 "linear_attention",722 "linear_attention",723 "full_attention",724 "linear_attention",725 "linear_attention",726 "linear_attention",727 "full_attention",728 "linear_attention",729 "linear_attention",730 "linear_attention",731 "full_attention"732 ],733 "linear_conv_kernel_dim": 4,734 "linear_key_head_dim": 128,735 "linear_num_key_heads": 16,736 "linear_num_value_heads": 32,737 "linear_value_head_dim": 128,738 "mamba_ssm_dtype": "float32",739 "max_position_embeddings": 262144,740 "model_type": "qwen3_5_moe_text",741 "moe_intermediate_size": 512,742 "mtp_num_hidden_layers": 1,743 "mtp_use_dedicated_embeddings": false,744 "num_attention_heads": 16,745 "num_experts": 256,746 "num_experts_per_tok": 8,747 "num_hidden_layers": 40,748 "num_key_value_heads": 2,749 "output_router_logits": false,750 "pad_token_id": 248044,751 "partial_rotary_factor": 0.25,752 "rms_norm_eps": 1e-06,753 "rope_parameters": {754 "mrope_interleaved": true,755 "mrope_section": [756 11,757 11,758 10759 ],760 "partial_rotary_factor": 0.25,761 "rope_theta": 10000000,762 "rope_type": "default"763 },764 "router_aux_loss_coef": 0.0,765 "shared_expert_intermediate_size": 512,766 "tie_word_embeddings": false,767 "use_cache": false,768 "vocab_size": 248320769 },770 "tie_word_embeddings": false,771 "top_k": 20,772 "top_p": 0.95,773 "transformers_version": "5.8.1",774 "video_token_id": 248057,775 "vision_config": {776 "deepstack_visual_indexes": [],777 "depth": 27,778 "dtype": "bfloat16",779 "hidden_act": "gelu_pytorch_tanh",780 "hidden_size": 1152,781 "in_channels": 3,782 "initializer_range": 0.02,783 "intermediate_size": 4304,784 "model_type": "qwen3_5_moe_vision",785 "num_heads": 16,786 "num_position_embeddings": 2304,787 "out_hidden_size": 2048,788 "patch_size": 16,789 "spatial_merge_size": 2,790 "temporal_patch_size": 2791 },792 "vision_end_token_id": 248054,793 "vision_start_token_id": 248053794}