amd/Llama-3.1-8B-Instruct-MXFP4-W4A4-MLCAL-C1000-GPTQ

pip install amd-quark==0.11.2 datasets accelerate evaluate nltk rouge-score lm-eval
cd Quark/examples/torch/language_modeling/llm_ptq/

python3 quantize_quark.py \
  --model_dir meta-llama/Llama-3.1-8B-Instruct \
  --model_attn_implementation sdpa \
  --quant_scheme mxfp4 \
  --quant_algo smoothquant,gptq \
  --quant_algo_config_file smoothquant smoothquant_a0.62.json \
  --dataset mlperf_cnn \
  --num_calib_data 1000 \
  --seq_len 2048 \
  --kv_cache_dtype fp8 --min_kv_scale 1.0 \
  --model_export hf_format \
  --export_weight_format real_quantized \
  --skip_evaluation \
  --output_dir Llama-3.1-8B-Instruct-MXFP4-W4A4-MLCAL-C1000-GPTQ

{
  "name": "smooth",
  "alpha": 0.62,
  "scale_clamp_min": 1e-3,
  "scaling_layers": [
    {"prev_op": "input_layernorm", "layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"], "inp": "self_attn.q_proj", "module2inspect": "self_attn"},
    {"prev_op": "self_attn.v_proj", "layers": ["self_attn.o_proj"], "inp": "self_attn.o_proj"},
    {"prev_op": "post_attention_layernorm", "layers": ["mlp.gate_proj", "mlp.up_proj"], "inp": "mlp.gate_proj", "module2inspect": "mlp"},
    {"prev_op": "mlp.up_proj", "layers": ["mlp.down_proj"], "inp": "mlp.down_proj"}
  ],
  "model_decoder_layers": "model.layers"
}