aday777/qwen4_exp_tiny_fixture

1{2  "architectures": [3    "Qwen4ExpForConditionalGeneration"4  ],5  "image_token_id": 248056,6  "language_model_only": true,7  "model_type": "qwen4_exp",8  "text_config": {9    "attention_bias": false,10    "attention_dropout": 0.0,11    "bos_token_id": 1,12    "dtype": "float32",13    "eos_token_id": 2,14    "full_attention_interval": 4,15    "hc_count": 2,16    "hc_lowrank": 32,17    "head_dim": 16,18    "heads_per_ngram": 2,19    "hidden_act": "silu",20    "hidden_size": 64,21    "indexer_budget": 32,22    "indexer_compress_ratio": 4,23    "indexer_head_dim": 16,24    "indexer_kv_heads": 1,25    "indexer_n_heads": 2,26    "initializer_range": 0.02,27    "layer_types": [28      "linear_attention",29      "linear_attention",30      "linear_attention",31      "full_attention"32    ],33    "linear_conv_kernel_dim": 4,34    "linear_key_head_dim": 16,35    "linear_num_key_heads": 2,36    "linear_num_value_heads": 4,37    "linear_value_head_dim": 16,38    "make_ngram_vocab_size_divisible_by": 128,39    "mamba_ssm_dtype": "float32",40    "max_position_embeddings": 256,41    "model_type": "qwen4_exp_text",42    "moe_intermediate_size": 32,43    "mtp": {44      "hybrid": true,45      "layer_types": [46        "full_attention"47      ],48      "mtp_use_hidden_state_from_layer": null,49      "num_hidden_layers": 1,50      "rope_theta": 1000000051    },52    "mtp_num_hidden_layers": 1,53    "mtp_use_dedicated_embeddings": false,54    "ngram_size": 3,55    "ngram_vocab_size_base": 4096,56    "num_attention_heads": 4,57    "num_experts": 8,58    "num_experts_per_tok": 2,59    "num_hidden_layers": 4,60    "num_key_value_heads": 2,61    "output_gate_type": "sigmoid",62    "output_router_logits": false,63    "pad_token_id": 0,64    "partial_rotary_factor": 0.25,65    "ple_conv_kernel_size": 4,66    "ple_embed_dim": 64,67    "ple_layer_ids": [68      269    ],70    "rms_norm_eps": 1e-06,71    "rope_parameters": {72      "mrope_interleaved": true,73      "mrope_section": [74        4,75        4,76        877      ],78      "partial_rotary_factor": 0.25,79      "rope_theta": 10000000,80      "rope_type": "default"81    },82    "router_aux_loss_coef": 0.001,83    "shared_expert_intermediate_size": 32,84    "split_ngram_parts": 128,85    "tie_word_embeddings": false,86    "use_cache": true,87    "vocab_size": 25688  },89  "tie_word_embeddings": false,90  "video_token_id": 248057,91  "vision_config": {92    "_note": "placeholder; this fixture ships NO vision/projector tensors",93    "model_type": "qwen4_exp"94  },95  "vision_end_token_id": 248054,96  "vision_start_token_id": 24805397}98