jerryzh168/phi4-float8dq

1{2  "architectures": [3    "Phi3ForCausalLM"4  ],5  "attention_bias": false,6  "attention_dropout": 0.0,7  "bos_token_id": 100257,8  "embd_pdrop": 0.0,9  "eos_token_id": 100265,10  "hidden_act": "silu",11  "hidden_size": 5120,12  "initializer_range": 0.02,13  "intermediate_size": 17920,14  "max_position_embeddings": 16384,15  "model_type": "phi3",16  "num_attention_heads": 40,17  "num_hidden_layers": 40,18  "num_key_value_heads": 10,19  "original_max_position_embeddings": 16384,20  "pad_token_id": 100349,21  "partial_rotary_factor": 1.0,22  "quantization_config": {23    "modules_to_not_convert": null,24    "quant_method": "torchao",25    "quant_type": {26      "default": {27        "_data": {28          "activation_dtype": {29            "_data": "float8_e4m3fn",30            "_type": "torch.dtype"31          },32          "granularity": {33            "_data": {},34            "_type": "PerRow",35            "_version": 136          },37          "mm_config": {38            "_data": {39              "emulate": false,40              "pad_inner_dim": false,41              "use_fast_accum": true42            },43            "_type": "Float8MMConfig",44            "_version": 145          },46          "set_inductor_config": true,47          "weight_dtype": {48            "_data": "float8_e4m3fn",49            "_type": "torch.dtype"50          }51        },52        "_type": "Float8DynamicActivationFloat8WeightConfig",53        "_version": 154      }55    },56    "quant_type_kwargs": {}57  },58  "resid_pdrop": 0.0,59  "rms_norm_eps": 1e-05,60  "rope_scaling": null,61  "rope_theta": 250000,62  "sliding_window": null,63  "tie_word_embeddings": false,64  "torch_dtype": "bfloat16",65  "transformers_version": "4.51.0.dev0",66  "use_cache": true,67  "vocab_size": 10035268}69