SmartManoj/Kimi-K3

1{2  "architectures": [3    "KimiK3ForConditionalGeneration"4  ],5  "auto_map": {6    "AutoConfig": "configuration_kimi_k3.KimiK3Config",7    "AutoModel": "modeling_kimi_k3.KimiK3ForConditionalGeneration",8    "AutoModelForCausalLM": "modeling_kimi_k3.KimiK3ForConditionalGeneration"9  },10  "bos_token_id": 163584,11  "dtype": "bfloat16",12  "eos_token_id": 163586,13  "ignore_index": -100,14  "image_placeholder": "<|kimi_image_placeholder|>",15  "media_placeholder_token_id": 163605,16  "model_type": "kimi_k3",17  "pad_token_id": 163839,18  "text_config": {19    "_name_or_path": "",20    "activation_situ_beta": 4.0,21    "activation_situ_linear_beta": 25.0,22    "add_cross_attention": false,23    "architectures": [24      "KimiLinearForCausalLM"25    ],26    "attn_res_block_size": 12,27    "auto_map": {28      "AutoConfig": "configuration_kimi_k3.KimiLinearConfig",29      "AutoModel": "modeling_kimi_linear.KimiLinearModel",30      "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM"31    },32    "bad_words_ids": null,33    "begin_suppress_tokens": null,34    "bos_token_id": 163584,35    "chunk_size_feed_forward": 0,36    "cross_attention_hidden_size": null,37    "decoder_start_token_id": null,38    "diversity_penalty": 0.0,39    "do_sample": false,40    "dtype": "bfloat16",41    "early_stopping": false,42    "encoder_no_repeat_ngram_size": 0,43    "eos_token_id": 163586,44    "exponential_decay_length_penalty": null,45    "finetuning_task": null,46    "first_k_dense_replace": 1,47    "forced_bos_token_id": null,48    "forced_eos_token_id": null,49    "hidden_act": "situ",50    "hidden_size": 7168,51    "id2label": {52      "0": "LABEL_0",53      "1": "LABEL_1"54    },55    "initializer_range": 0.02,56    "intermediate_size": 33792,57    "is_decoder": false,58    "is_encoder_decoder": false,59    "kv_lora_rank": 512,60    "label2id": {61      "LABEL_0": 0,62      "LABEL_1": 163    },64    "latent_moe_use_norm": true,65    "length_penalty": 1.0,66    "linear_attn_config": {67      "full_attn_layers": [68        4,69        8,70        12,71        16,72        20,73        24,74        28,75        32,76        36,77        40,78        44,79        48,80        52,81        56,82        60,83        64,84        68,85        72,86        76,87        80,88        84,89        88,90        92,91        9392      ],93      "gate_lower_bound": -5.0,94      "head_dim": 128,95      "kda_layers": [96        1,97        2,98        3,99        5,100        6,101        7,102        9,103        10,104        11,105        13,106        14,107        15,108        17,109        18,110        19,111        21,112        22,113        23,114        25,115        26,116        27,117        29,118        30,119        31,120        33,121        34,122        35,123        37,124        38,125        39,126        41,127        42,128        43,129        45,130        46,131        47,132        49,133        50,134        51,135        53,136        54,137        55,138        57,139        58,140        59,141        61,142        62,143        63,144        65,145        66,146        67,147        69,148        70,149        71,150        73,151        74,152        75,153        77,154        78,155        79,156        81,157        82,158        83,159        85,160        86,161        87,162        89,163        90,164        91165      ],166      "num_heads": 96,167      "short_conv_kernel_size": 4,168      "use_full_rank_gate": true169    },170    "max_length": 20,171    "max_position_embeddings": 1048576,172    "min_length": 0,173    "mla_use_nope": true,174    "mla_use_output_gate": true,175    "model_type": "kimi_linear",176    "moe_intermediate_size": 3072,177    "moe_layer_freq": 1,178    "moe_renormalize": true,179    "moe_router_activation_func": "sigmoid",180    "no_repeat_ngram_size": 0,181    "num_attention_heads": 96,182    "num_beam_groups": 1,183    "num_beams": 1,184    "num_expert_group": 1,185    "num_experts": 896,186    "num_experts_per_token": 16,187    "num_hidden_layers": 93,188    "num_key_value_heads": 96,189    "num_nextn_predict_layers": 0,190    "num_return_sequences": 1,191    "num_shared_experts": 2,192    "output_attentions": false,193    "output_hidden_states": false,194    "output_scores": false,195    "pad_token_id": 163839,196    "prefix": null,197    "problem_type": null,198    "pruned_heads": {},199    "q_lora_rank": 1536,200    "qk_nope_head_dim": 128,201    "qk_rope_head_dim": 64,202    "quantization_config": {203      "config_groups": {204        "group_0": {205          "format": "mxfp4-pack-quantized",206          "input_activations": null,207          "output_activations": null,208          "targets": [209            "Linear"210          ],211          "weights": {212            "actorder": null,213            "block_structure": null,214            "dynamic": false,215            "group_size": 32,216            "num_bits": 4,217            "observer": "minmax",218            "observer_kwargs": {},219            "scale_dtype": "torch.uint8",220            "strategy": "group",221            "symmetric": true,222            "type": "float",223            "zp_dtype": null224          }225        }226      },227      "format": "mxfp4-pack-quantized",228      "global_compression_ratio": null,229      "ignore": [230        "re:.*self_attn.*",231        "re:.*shared_experts.*",232        "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",233        "re:.*lm_head.*",234        "re:.*vision_tower.*",235        "re:.*mm_projector.*"236      ],237      "kv_cache_scheme": null,238      "quant_method": "compressed-tensors",239      "quantization_status": "compressed"240    },241    "remove_invalid_values": false,242    "repetition_penalty": 1.0,243    "return_dict": true,244    "return_dict_in_generate": false,245    "rms_norm_eps": 1e-05,246    "routed_expert_hidden_size": 3584,247    "routed_scaling_factor": 1.0,248    "sep_token_id": null,249    "suppress_tokens": null,250    "task_specific_params": null,251    "temperature": 1.0,252    "tf_legacy_loss": false,253    "tie_encoder_decoder": false,254    "tie_word_embeddings": false,255    "tokenizer_class": null,256    "top_k": 50,257    "top_p": 1.0,258    "topk_group": 1,259    "topk_method": "noaux_tc",260    "torchscript": false,261    "transformers_version": "4.56.2",262    "typical_p": 1.0,263    "use_bfloat16": false,264    "use_cache": true,265    "use_grouped_topk": true,266    "v_head_dim": 128,267    "vocab_size": 163840268  },269  "tie_word_embeddings": false,270  "vision_config": {271    "_attn_implementation": "flash_attention_2",272    "activation_func": "gelu_pytorch_tanh",273    "attn_bias": false,274    "init_pos_emb_height": 64,275    "init_pos_emb_time": 4,276    "init_pos_emb_width": 64,277    "linear_bias": false,278    "merge_kernel_size": [279      2,280      2281    ],282    "merge_type": "sd2_tpool",283    "mlp_type": "mlp2",284    "mm_hidden_size": 1024,285    "mm_projector_type": "patchmergerv2",286    "norm_type": "rmsnorm",287    "patch_embed_proj_bias": false,288    "patch_size": 14,289    "pos_emb_interpolation_mode": "bilinear",290    "pos_emb_type": "divided_fixed",291    "projector_hidden_act": "gelu",292    "projector_ln_eps": 1e-05,293    "qkv_hidden_size": 1536,294    "text_hidden_size": 7168,295    "vt_hidden_size": 1024,296    "vt_intermediate_size": 4096,297    "vt_num_attention_heads": 12,298    "vt_num_hidden_layers": 27299  }300}