Shoolife/Phi-4-mini-instruct-TensorRT-LLM-Checkpoint-FP8

1{2    "producer": {3        "name": "modelopt",4        "version": "0.37.0"5    },6    "architecture": "Phi3ForCausalLM",7    "dtype": "bfloat16",8    "logits_dtype": "float16",9    "num_hidden_layers": 32,10    "num_attention_heads": 24,11    "num_key_value_heads": 8,12    "hidden_size": 3072,13    "norm_epsilon": 1e-05,14    "vocab_size": 200064,15    "max_position_embeddings": 131072,16    "hidden_act": "swiglu",17    "use_parallel_embedding": true,18    "embedding_sharding_dim": 0,19    "head_size": 128,20    "intermediate_size": 8192,21    "position_embedding_type": "long_rope",22    "share_embedding_table": false,23    "residual_mlp": false,24    "bias": false,25    "rotary_pct": 0.75,26    "rank": 0,27    "decoder": "phi3",28    "rmsnorm": true,29    "lm_head_bias": false,30    "rotary_base": 10000.0,31    "rotary_scaling": null,32    "runtime_defaults": null,33    "mapping": {34        "world_size": 1,35        "gpus_per_node": 8,36        "cp_size": 1,37        "tp_size": 1,38        "pp_size": 1,39        "moe_tp_size": 1,40        "moe_cluster_size": 1,41        "moe_ep_size": 1,42        "attn_tp_size": 1,43        "attn_cp_size": 1,44        "cp_config": {},45        "enable_attention_dp": false,46        "enable_lm_head_tp_in_adp": false47    },48    "quantization": {49        "quant_algo": "FP8",50        "kv_cache_quant_algo": "FP8",51        "group_size": 128,52        "smoothquant_val": 0.5,53        "clamp_val": null,54        "use_meta_recipe": false,55        "has_zero_point": false,56        "pre_quant_scale": false,57        "exclude_modules": [58            "lm_head"59        ],60        "mamba_ssm_cache_dtype": null61    },62    "qk_layernorm": false,63    "rotary_embedding_dim": 96,64    "tie_word_embeddings": true,65    "original_max_position_embeddings": 4096,66    "longrope_scaling_short_factors": [67        1.0,68        1.0,69        1.0,70        1.0,71        1.0,72        1.0,73        1.0,74        1.0,75        1.0,76        1.0,77        1.0,78        1.0,79        1.0,80        1.0,81        1.0,82        1.0,83        1.0,84        1.0,85        1.0,86        1.0,87        1.0,88        1.0,89        1.0,90        1.0,91        1.0,92        1.0,93        1.0,94        1.0,95        1.0,96        1.0,97        1.0,98        1.0,99        1.0,100        1.0,101        1.0,102        1.0,103        1.0,104        1.0,105        1.0,106        1.0,107        1.0,108        1.0,109        1.0,110        1.0,111        1.0,112        1.0,113        1.0,114        1.0115    ],116    "longrope_scaling_long_factors": [117        1,118        1.118320672,119        1.250641126,120        1.398617824,121        1.564103225,122        1.74916897,123        1.956131817,124        2.187582649,125        2.446418898,126        2.735880826,127        3.059592084,128        3.421605075,129        3.826451687,130        4.279200023,131        4.785517845,132        5.351743533,133        5.984965424,134        6.693110555,135        7.485043894,136        8.370679318,137        9.36110372,138        10.4687158,139        11.70738129,140        13.09260651,141        14.64173252,142        16.37415215,143        18.31155283,144        20.47818807,145        22.90118105,146        25.61086418,147        28.64115884,148        32.03,149        32.1,150        32.13,151        32.23,152        32.6,153        32.61,154        32.64,155        32.66,156        32.7,157        32.71,158        32.93,159        32.97,160        33.28,161        33.49,162        33.5,163        44.16,164        47.77165    ],166    "model_type": "phi3"167}