aday777/qwen4_exp_tiny_fixture

1#!/usr/bin/env python32"""Build a tiny, deterministic random-init qwen4_exp TEXT fixture (stdlib only).3 4Purpose: Qwen/Qwen3.8-Flash-Next (released 2026-08-24) is a large multimodal5Qwen4ExpForConditionalGeneration MoE with a hybrid linear+full attention stack,6ngram vocab, an indexer, PLE, hyper-connections, and an MTP head, so nobody can7instantiate it in CI or on a laptop. This fixture ships a ~0.3 MB random-init8TEXT checkpoint plus a reduced config that keeps the real qwen4_exp / qwen4_exp_text9field names (layer_types, linear_*, ngram_*, indexer_*, ple_*, hc_*, mtp,10rope_parameters, vision_config) so loaders, quant planners, and CI jobs can11exercise the new architecture's config parsing, expert-table sizing, and12safetensors load path without the real weights.13 14Random-init: NOT trained and NOT a quality claim. The tensor set is a REDUCED15standard-attention + MoE convention; the real model's linear-attention (conv/ssm),16ngram, indexer, PLE, hyper-connection, MTP, and vision/projector tensors are NOT17included (see README omissions). Geometry is documented in the README.18"""19import hashlib20import json21import math22import os23import struct24 25M64 = (1 << 64) - 126SEED = 2026090327SCALE = 0.0228 29# ---- tiny geometry (reduced from the real text_config, documented in README) ----30VOCAB = 25631HIDDEN = 6432LAYERS = 433HEADS = 434KV_HEADS = 235HEAD_DIM = 1636MOE_INTER = 3237SHARED_INTER = 3238NUM_EXPERTS = 839TOPK = 240 41 42class SplitMix64:43    """SplitMix64 + Box-Muller, identical to the llama/t5/glm fixtures."""44 45    def __init__(self, seed):46        self.state = seed & M6447        self._spare = None48 49    def next_u64(self):50        self.state = (self.state + 0x9E3779B97F4A7C15) & M6451        z = self.state52        z = ((z ^ (z >> 30)) * 0xBF584A7F17C119E3) & M6453        z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M6454        return z ^ (z >> 31)55 56    def uniform(self):57        return (self.next_u64() >> 11) / float(1 << 53)58 59    def gauss(self):60        if self._spare is not None:61            value, self._spare = self._spare, None62            return value63        u1 = 1.0 - self.uniform()64        u2 = self.uniform()65        radius = math.sqrt(-2.0 * math.log(u1))66        theta = 2.0 * math.pi * u267        self._spare = radius * math.sin(theta)68        return radius * math.cos(theta)69 70 71def build_tensors():72    shapes = {73        "model.embed_tokens.weight": (VOCAB, HIDDEN),74        "model.norm.weight": (HIDDEN,),75    }76    ones = {"model.norm.weight"}77    for layer in range(LAYERS):78        p = "model.layers.%d." % layer79        shapes[p + "input_layernorm.weight"] = (HIDDEN,)80        shapes[p + "post_attention_layernorm.weight"] = (HIDDEN,)81        ones.add(p + "input_layernorm.weight")82        ones.add(p + "post_attention_layernorm.weight")83        shapes[p + "self_attn.q_proj.weight"] = (HEADS * HEAD_DIM, HIDDEN)84        shapes[p + "self_attn.k_proj.weight"] = (KV_HEADS * HEAD_DIM, HIDDEN)85        shapes[p + "self_attn.v_proj.weight"] = (KV_HEADS * HEAD_DIM, HIDDEN)86        shapes[p + "self_attn.o_proj.weight"] = (HIDDEN, HEADS * HEAD_DIM)87        shapes[p + "mlp.gate.weight"] = (NUM_EXPERTS, HIDDEN)88        for expert in range(NUM_EXPERTS):89            e = p + "mlp.experts.%d." % expert90            shapes[e + "gate_proj.weight"] = (MOE_INTER, HIDDEN)91            shapes[e + "up_proj.weight"] = (MOE_INTER, HIDDEN)92            shapes[e + "down_proj.weight"] = (HIDDEN, MOE_INTER)93        shapes[p + "mlp.shared_expert.gate_proj.weight"] = (SHARED_INTER, HIDDEN)94        shapes[p + "mlp.shared_expert.up_proj.weight"] = (SHARED_INTER, HIDDEN)95        shapes[p + "mlp.shared_expert.down_proj.weight"] = (HIDDEN, SHARED_INTER)96 97    rng = SplitMix64(SEED)98    out = {}99    for name in sorted(shapes):100        shape = shapes[name]101        count = 1102        for dim in shape:103            count *= dim104        if name in ones:105            values = [1.0] * count106        else:107            values = [rng.gauss() * SCALE for _ in range(count)]108        blob = b"".join(109            struct.pack("<f", struct.unpack("<f", struct.pack("<f", v))[0]) for v in values110        )111        out[name] = (list(shape), "F32", blob)112    return out113 114 115def write_safetensors(path, tensors, metadata):116    header = {"__metadata__": metadata}117    offset = 0118    blobs = []119    for name in sorted(tensors):120        shape, dtype, blob = tensors[name]121        header[name] = {"dtype": dtype, "shape": shape,122                        "data_offsets": [offset, offset + len(blob)]}123        offset += len(blob)124        blobs.append(blob)125    raw = json.dumps(header, separators=(",", ":")).encode("utf-8")126    raw += b" " * ((-len(raw)) % 8)127    with open(path, "wb") as handle:128        handle.write(struct.pack("<Q", len(raw)))129        handle.write(raw)130        for blob in blobs:131            handle.write(blob)132    return len(raw), offset133 134 135def main():136    out_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),137                           "qwen4_exp_tiny_fixture")138    os.makedirs(out_dir, exist_ok=True)139 140    tensors = build_tensors()141    metadata = {142        "format": "pt",143        "source": "usefulHuggingface",144        "generator": "SplitMix64 seed=%d Box-Muller scale=%s float32 row-major" % (SEED, SCALE),145    }146    header_len, data_len = write_safetensors(147        os.path.join(out_dir, "model.safetensors"), tensors, metadata)148 149    layer_types = ["linear_attention", "linear_attention", "linear_attention",150                   "full_attention"]151    text_config = {152        "attention_bias": False,153        "attention_dropout": 0.0,154        "bos_token_id": 1,155        "dtype": "float32",156        "eos_token_id": 2,157        "full_attention_interval": 4,158        "hc_count": 2,159        "hc_lowrank": 32,160        "head_dim": HEAD_DIM,161        "heads_per_ngram": 2,162        "hidden_act": "silu",163        "hidden_size": HIDDEN,164        "indexer_budget": 32,165        "indexer_compress_ratio": 4,166        "indexer_head_dim": 16,167        "indexer_kv_heads": 1,168        "indexer_n_heads": 2,169        "initializer_range": 0.02,170        "layer_types": layer_types,171        "linear_conv_kernel_dim": 4,172        "linear_key_head_dim": 16,173        "linear_num_key_heads": 2,174        "linear_num_value_heads": 4,175        "linear_value_head_dim": 16,176        "make_ngram_vocab_size_divisible_by": 128,177        "mamba_ssm_dtype": "float32",178        "max_position_embeddings": 256,179        "model_type": "qwen4_exp_text",180        "moe_intermediate_size": MOE_INTER,181        "mtp": {182            "hybrid": True,183            "layer_types": ["full_attention"],184            "mtp_use_hidden_state_from_layer": None,185            "num_hidden_layers": 1,186            "rope_theta": 10000000,187        },188        "mtp_num_hidden_layers": 1,189        "mtp_use_dedicated_embeddings": False,190        "ngram_size": 3,191        "ngram_vocab_size_base": 4096,192        "num_attention_heads": HEADS,193        "num_experts": NUM_EXPERTS,194        "num_experts_per_tok": TOPK,195        "num_hidden_layers": LAYERS,196        "num_key_value_heads": KV_HEADS,197        "output_gate_type": "sigmoid",198        "output_router_logits": False,199        "pad_token_id": 0,200        "partial_rotary_factor": 0.25,201        "ple_conv_kernel_size": 4,202        "ple_embed_dim": HIDDEN,203        "ple_layer_ids": [2],204        "rms_norm_eps": 1e-6,205        "rope_parameters": {206            "mrope_interleaved": True,207            "mrope_section": [4, 4, 8],208            "partial_rotary_factor": 0.25,209            "rope_theta": 10000000,210            "rope_type": "default",211        },212        "router_aux_loss_coef": 0.001,213        "shared_expert_intermediate_size": SHARED_INTER,214        "split_ngram_parts": 128,215        "tie_word_embeddings": False,216        "use_cache": True,217        "vocab_size": VOCAB,218    }219    config = {220        "architectures": ["Qwen4ExpForConditionalGeneration"],221        "image_token_id": 248056,222        "language_model_only": True,223        "model_type": "qwen4_exp",224        "text_config": text_config,225        "tie_word_embeddings": False,226        "video_token_id": 248057,227        "vision_config": {228            "model_type": "qwen4_exp",229            "_note": "placeholder; this fixture ships NO vision/projector tensors",230        },231        "vision_end_token_id": 248054,232        "vision_start_token_id": 248053,233    }234    with open(os.path.join(out_dir, "config.json"), "w") as handle:235        json.dump(config, handle, indent=2, sort_keys=True)236        handle.write("\n")237 238    with open(os.path.join(out_dir, "generation_config.json"), "w") as handle:239        json.dump({"bos_token_id": 1, "eos_token_id": 2, "pad_token_id": 0,240                   "no_repeat_ngram_size": 4, "seed": SEED},241                  handle, indent=2, sort_keys=True)242        handle.write("\n")243 244    with open(os.path.join(out_dir, "tokenizer_config.json"), "w") as handle:245        json.dump({"model_max_length": 256, "bos_token": "<s>", "eos_token": "</s>",246                   "unk_token": "<unk>", "pad_token": "<pad>",247                   "model_input_names": ["input_ids"]},248                  handle, indent=2, sort_keys=True)249        handle.write("\n")250 251    with open(os.path.join(out_dir, "special_tokens_map.json"), "w") as handle:252        json.dump({"additional_special_tokens": ["<pad>", "<unk>"],253                   "bos_token": "<s>", "eos_token": "</s>",254                   "pad_token": "<pad>", "unk_token": "<unk>"},255                  handle, indent=2, sort_keys=True)256        handle.write("\n")257 258    lines = []259    for name in sorted(tensors):260        shape, dtype, blob = tensors[name]261        lines.append("%s  %s  %s  %d  %s" % (name, dtype, "x".join(map(str, shape)),262                                             len(blob), hashlib.sha256(blob).hexdigest()))263    with open(os.path.join(out_dir, "checksums.txt"), "w") as handle:264        handle.write("\n".join(lines) + "\n")265 266    print("header_len=%d data_len=%d tensors=%d" % (header_len, data_len, len(tensors)))267    print("total_params=%d" % (data_len // 4))268 269 270if __name__ == "__main__":271    main()272