inference-optimization/DSV4-tiny-empty

1{2  "architectures": [3    "DeepseekV4ForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "bos_token_id": 0,8  "compress_rates": {9    "compressed_sparse_attention": 4,10    "heavily_compressed_attention": 12811  },12  "compress_rope_theta": 160000,13  "dtype": "bfloat16",14  "eos_token_id": 1,15  "expert_dtype": "fp4",16  "hc_eps": 1e-06,17  "hc_mult": 4,18  "hc_sinkhorn_iters": 20,19  "head_dim": 512,20  "hidden_act": "silu",21  "hidden_size": 4096,22  "index_head_dim": 128,23  "index_n_heads": 64,24  "index_topk": 512,25  "initializer_range": 0.02,26  "layer_types": [27    "heavily_compressed_attention",28    "compressed_sparse_attention",29    "sliding_attention"30  ],31  "max_position_embeddings": 1048576,32  "mlp_bias": false,33  "mlp_layer_types": [34    "hash_moe",35    "moe",36    "moe"37  ],38  "model_type": "deepseek_v4",39  "moe_intermediate_size": 2048,40  "n_routed_experts": 16,41  "n_shared_experts": 1,42  "norm_topk_prob": true,43  "num_attention_heads": 64,44  "num_experts_per_tok": 6,45  "num_hidden_layers": 3,46  "num_key_value_heads": 1,47  "num_nextn_predict_layers": 0,48  "o_groups": 8,49  "o_lora_rank": 1024,50  "output_router_logits": false,51  "pad_token_id": null,52  "partial_rotary_factor": 0.125,53  "q_lora_rank": 1024,54  "qk_rope_head_dim": 64,55  "rms_norm_eps": 1e-06,56  "rope_parameters": {57    "compress": {58      "attention_factor": 1.0,59      "beta_fast": 32,60      "beta_slow": 1,61      "factor": 16,62      "original_max_position_embeddings": 65536,63      "partial_rotary_factor": 0.125,64      "rope_theta": 160000,65      "rope_type": "yarn",66      "type": "yarn"67    },68    "main": {69      "partial_rotary_factor": 0.125,70      "rope_theta": 10000,71      "rope_type": "default"72    },73    "partial_rotary_factor": 0.125,74    "rope_theta": 10000,75    "rope_type": "default"76  },77  "rope_theta": 10000,78  "routed_scaling_factor": 1.5,79  "router_aux_loss_coef": 0.001,80  "router_jitter_noise": 0.0,81  "scoring_func": "sqrtsoftplus",82  "sliding_window": 128,83  "swiglu_limit": 10.0,84  "tie_word_embeddings": false,85  "topk_method": "noaux_tc",86  "transformers_version": "5.8.0.dev0",87  "use_cache": true,88  "vocab_size": 12928089}90