unsloth/Kimi-K2.7-Code

1{2  "architectures": [3    "KimiK25ForConditionalGeneration"4  ],5  "auto_map": {6    "AutoConfig": "configuration_kimi_k25.KimiK25Config",7    "AutoModel": "modeling_kimi_k25.KimiK25ForConditionalGeneration",8    "AutoModelForCausalLM": "modeling_kimi_k25.KimiK25ForConditionalGeneration"9  },10  "bos_token_id": 163584,11  "torch_dtype": "bfloat16",12  "eos_token_id": 163586,13  "ignore_index": -100,14  "media_placeholder_token_id": 163605,15  "model_type": "kimi_k25",16  "pad_token_id": 163839,17  "quantization_config": {18    "config_groups": {19      "group_0": {20        "input_activations": null,21        "output_activations": null,22        "targets": [23          "Linear"24        ],25        "weights": {26          "actorder": null,27          "block_structure": null,28          "dynamic": false,29          "group_size": 32,30          "num_bits": 4,31          "observer": "minmax",32          "observer_kwargs": {},33          "strategy": "group",34          "symmetric": true,35          "type": "int"36        }37      }38    },39    "format": "pack-quantized",40    "ignore": [41      "re:.*self_attn.*",42      "re:.*shared_experts.*",43      "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",44      "re:.*lm_head.*",45      "re:.*vision_tower.*",46      "re:.*mm_projector.*"47    ],48    "kv_cache_scheme": null,49    "quant_method": "compressed-tensors",50    "quantization_status": "compressed"51  },52  "text_config": {53    "_name_or_path": "",54    "add_cross_attention": false,55    "architectures": [56      "DeepseekV3ForCausalLM"57    ],58    "attention_bias": false,59    "attention_dropout": 0.0,60    "auto_map": {61      "AutoConfig": "configuration_deepseek.DeepseekV3Config",62      "AutoModel": "modeling_deepseek.DeepseekV3Model",63      "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"64    },65    "aux_loss_alpha": 0.001,66    "bos_token_id": 163584,67    "chunk_size_feed_forward": 0,68    "cross_attention_hidden_size": null,69    "decoder_start_token_id": null,70    "torch_dtype": "bfloat16",71    "eos_token_id": 163586,72    "ep_size": 1,73    "finetuning_task": null,74    "first_k_dense_replace": 1,75    "hidden_act": "silu",76    "hidden_size": 7168,77    "id2label": {78      "0": "LABEL_0",79      "1": "LABEL_1"80    },81    "initializer_range": 0.02,82    "intermediate_size": 18432,83    "is_decoder": false,84    "is_encoder_decoder": false,85    "kv_lora_rank": 512,86    "label2id": {87      "LABEL_0": 0,88      "LABEL_1": 189    },90    "max_position_embeddings": 262144,91    "model_type": "deepseek_v3",92    "moe_intermediate_size": 2048,93    "moe_layer_freq": 1,94    "n_group": 1,95    "n_routed_experts": 384,96    "n_shared_experts": 1,97    "norm_topk_prob": true,98    "num_attention_heads": 64,99    "num_experts_per_tok": 8,100    "num_hidden_layers": 61,101    "num_key_value_heads": 64,102    "num_nextn_predict_layers": 0,103    "output_attentions": false,104    "output_hidden_states": false,105    "pad_token_id": 163839,106    "prefix": null,107    "pretraining_tp": 1,108    "problem_type": null,109    "pruned_heads": {},110    "q_lora_rank": 1536,111    "qk_nope_head_dim": 128,112    "qk_rope_head_dim": 64,113    "quantization_config": {114      "config_groups": {115        "group_0": {116          "input_activations": null,117          "output_activations": null,118          "targets": [119            "Linear"120          ],121          "weights": {122            "actorder": null,123            "block_structure": null,124            "dynamic": false,125            "group_size": 32,126            "num_bits": 4,127            "observer": "minmax",128            "observer_kwargs": {},129            "strategy": "group",130            "symmetric": true,131            "type": "int"132          }133        }134      },135      "format": "pack-quantized",136      "ignore": [137        "re:.*self_attn.*",138        "re:.*shared_experts.*",139        "re:.*mlp\\.(gate|up|gate_up|down)_proj.*",140        "re:.*lm_head.*",141        "re:.*vision_tower.*",142        "re:.*mm_projector.*"143      ],144      "kv_cache_scheme": null,145      "quant_method": "compressed-tensors",146      "quantization_status": "compressed"147    },148    "return_dict": true,149    "rms_norm_eps": 1e-05,150    "rope_parameters": {151      "beta_fast": 32.0,152      "beta_slow": 1.0,153      "factor": 64.0,154      "mscale": 1.0,155      "mscale_all_dim": 1.0,156      "original_max_position_embeddings": 4096,157      "rope_theta": 50000.0,158      "rope_type": "yarn",159      "type": "yarn"160    },161    "rope_theta": 50000.0,162    "routed_scaling_factor": 2.827,163    "scoring_func": "sigmoid",164    "sep_token_id": null,165    "seq_aux": true,166    "task_specific_params": null,167    "tf_legacy_loss": false,168    "tie_encoder_decoder": false,169    "tie_word_embeddings": false,170    "tokenizer_class": null,171    "topk_group": 1,172    "topk_method": "noaux_tc",173    "torchscript": false,174    "use_bfloat16": false,175    "use_cache": true,176    "v_head_dim": 128,177    "vocab_size": 163840178  },179  "tie_word_embeddings": false,180  "transformers_version": "5.13.0.dev0",181  "unsloth_fixed": true,182  "use_unified_vision_chunk": true,183  "video_placeholder": "<|kimi_k25_video_placeholder|>",184  "vision_config": {185    "init_pos_emb_height": 64,186    "init_pos_emb_time": 4,187    "init_pos_emb_width": 64,188    "merge_kernel_size": [189      2,190      2191    ],192    "merge_type": "sd2_tpool",193    "mm_hidden_size": 1152,194    "mm_projector_type": "patchmerger",195    "model_type": "",196    "patch_size": 14,197    "pos_emb_type": "divided_fixed",198    "projector_hidden_act": "gelu",199    "projector_ln_eps": 1e-05,200    "text_hidden_size": 7168,201    "video_attn_type": "spatial_temporal",202    "vt_hidden_size": 1152,203    "vt_intermediate_size": 4304,204    "vt_num_attention_heads": 16,205    "vt_num_hidden_layers": 27206  }207}