nm-testing/Devstral-Small-2505-FP8-dynamic

1{2  "_name_or_path": "/mnt/llms/models/mistralai/Devstral-Small-2505",3  "architectures": [4    "MistralForCausalLM"5  ],6  "attention_dropout": 0.0,7  "bos_token_id": 1,8  "eos_token_id": 2,9  "head_dim": 128,10  "hidden_act": "silu",11  "hidden_size": 5120,12  "initializer_range": 0.02,13  "intermediate_size": 32768,14  "max_position_embeddings": 131072,15  "model_type": "mistral",16  "num_attention_heads": 32,17  "num_hidden_layers": 40,18  "num_key_value_heads": 8,19  "quantization_config": {20    "config_groups": {21      "group_0": {22        "input_activations": {23          "actorder": null,24          "block_structure": null,25          "dynamic": true,26          "group_size": null,27          "num_bits": 8,28          "observer": null,29          "observer_kwargs": {},30          "strategy": "token",31          "symmetric": true,32          "type": "float"33        },34        "output_activations": null,35        "targets": [36          "Linear"37        ],38        "weights": {39          "actorder": null,40          "block_structure": null,41          "dynamic": false,42          "group_size": null,43          "num_bits": 8,44          "observer": "minmax",45          "observer_kwargs": {},46          "strategy": "channel",47          "symmetric": true,48          "type": "float"49        }50      }51    },52    "format": "float-quantized",53    "global_compression_ratio": null,54    "ignore": [55      "lm_head"56    ],57    "kv_cache_scheme": null,58    "quant_method": "compressed-tensors",59    "quantization_status": "compressed"60  },61  "rms_norm_eps": 1e-05,62  "rope_theta": 1000000000.0,63  "sliding_window": null,64  "tie_word_embeddings": false,65  "torch_dtype": "bfloat16",66  "transformers_version": "4.49.0",67  "use_cache": true,68  "vocab_size": 13107269}