peft-internal-testing/tiny-random-gemma4-E2B

1---2library_name: transformers3tags: []4---5 6# Model Card for Model ID7 8The code to create this checkpoint is based on https://huggingface.co/tiny-random/gemma-4-e with a few small changes:9 10```python11import json12import os13 14import torch15from huggingface_hub import hf_hub_download16 17from transformers import (18    AutoConfig,19    AutoProcessor,20    Gemma4ForConditionalGeneration,21    set_seed,22)23 24source_model_id = "google/gemma-4-E4B"25save_folder = "/tmp/peft/tiny-random-gemma4"26 27processor = AutoProcessor.from_pretrained(source_model_id)28 29 30with open(31    hf_hub_download(source_model_id, filename="config.json", repo_type="model"), "r", encoding="utf-8",32) as f:33    config_json = json.load(f)34 35config_json["audio_config"].update(36    {37        "num_attention_heads": 2,38        "num_hidden_layers": 2,39        "hidden_size": 64,40        "output_proj_dims": 32,41    }42)43config_json["text_config"].update(44    {45        "global_head_dim": 64,46        "head_dim": 32,47        "hidden_size": 8,48        "hidden_size_per_layer_input": 2,49        "intermediate_size": 64,50        "layer_types": [51            "sliding_attention",52            "full_attention",53            "sliding_attention",54            "full_attention",55        ],56        "num_attention_heads": 8,57        "num_hidden_layers": 4,58        "num_key_value_heads": 4,59        "num_kv_shared_layers": 2,60    }61)62config_json["vision_config"].update(63    {64        "num_hidden_layers": 2,65        "hidden_size": 8,66        "intermediate_size": 64,67        "head_dim": 32,68        "global_head_dim": 32,69        "num_attention_heads": 4,70        "num_key_value_heads": 4,71    }72)73 74with open(f"{save_folder}/config.json", "w", encoding="utf-8") as f:75    json.dump(config_json, f, indent=2)76config = AutoConfig.from_pretrained(save_folder)77 78torch.set_default_dtype(torch.bfloat16)79model = Gemma4ForConditionalGeneration(config)80torch.set_default_dtype(torch.float32)81set_seed(42)82model = model.cpu()83 84all_numels = 085for name, p in sorted(model.named_parameters()):86    all_numels += p.numel()87with torch.no_grad():88    for name, p in sorted(model.named_parameters()):89        torch.nn.init.normal_(p, 0, 0.2)90        print(name, p.shape, f"{p.numel() / all_numels * 100: .4f}%")91 92 93token = os.environ.get("HF_TOKEN")94processor.push_to_hub("peft-internal-testing/tiny-random-gemma4-E2B", token=token)95model.push_to_hub("peft-internal-testing/tiny-random-gemma4-E2B", token=token)96```