RedHatAI/GLM-4.6-quantized.w8a8

1{2  "architectures": [3    "Glm4MoeForCausalLM"4  ],5  "attention_bias": true,6  "attention_dropout": 0.0,7  "dtype": "bfloat16",8  "eos_token_id": [9    151329,10    151336,11    15133812  ],13  "first_k_dense_replace": 3,14  "head_dim": 128,15  "hidden_act": "silu",16  "hidden_size": 5120,17  "initializer_range": 0.02,18  "intermediate_size": 12288,19  "max_position_embeddings": 202752,20  "model_type": "glm4_moe",21  "moe_intermediate_size": 1536,22  "n_group": 1,23  "n_routed_experts": 160,24  "n_shared_experts": 1,25  "norm_topk_prob": true,26  "num_attention_heads": 96,27  "num_experts_per_tok": 8,28  "num_hidden_layers": 92,29  "num_key_value_heads": 8,30  "num_nextn_predict_layers": 1,31  "pad_token_id": 151329,32  "partial_rotary_factor": 0.5,33  "quantization_config": {34    "config_groups": {35      "group_0": {36        "format": "int-quantized",37        "input_activations": {38          "actorder": null,39          "block_structure": null,40          "dynamic": true,41          "group_size": null,42          "num_bits": 8,43          "observer": null,44          "observer_kwargs": {},45          "strategy": "token",46          "symmetric": true,47          "type": "int"48        },49        "output_activations": null,50        "targets": [51          "Linear"52        ],53        "weights": {54          "actorder": null,55          "block_structure": null,56          "dynamic": false,57          "group_size": null,58          "num_bits": 8,59          "observer": "minmax",60          "observer_kwargs": {},61          "strategy": "channel",62          "symmetric": true,63          "type": "int"64        }65      }66    },67    "format": "int-quantized",68    "global_compression_ratio": null,69    "ignore": [70      "lm_head"71    ],72    "kv_cache_scheme": null,73    "quant_method": "compressed-tensors",74    "quantization_status": "compressed",75    "sparsity_config": {},76    "transform_config": {},77    "version": "0.12.3.dev31+gb019b89.d20251216"78  },79  "rms_norm_eps": 1e-05,80  "rope_scaling": null,81  "rope_theta": 1000000,82  "routed_scaling_factor": 2.5,83  "tie_word_embeddings": false,84  "topk_group": 1,85  "transformers_version": "4.57.3",86  "use_cache": true,87  "use_qk_norm": true,88  "vocab_size": 15155289}