optimum-intel-internal-testing/tiny-random-smollm3

1---2license: apache-2.03---4 5```python6 7from transformers import (8    AutoTokenizer,9    SmolLM3Config,10    SmolLM3ForCausalLM,11)12 13def create_tiny_random_smollm3():14 15    config = SmolLM3Config(16        vocab_size=128256,17        hidden_size=32,18        intermediate_size=64,19        num_hidden_layers=2,20        num_attention_heads=4,21        num_key_value_heads=2,22        max_position_embeddings=256,23        hidden_act="silu",24        rms_norm_eps=1e-6,25        tie_word_embeddings=True,26        use_cache=True,27        attention_bias=False,28        mlp_bias=False,29        use_sliding_window=False,30        pad_token_id=128004,31        bos_token_id=128000,32        eos_token_id=128012,33    )34 35    model = SmolLM3ForCausalLM(config)36    print(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")37 38    # Use SmolLM3-3B tokenizer39    tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM3-3B")40 41    output_dir = "./tiny-random-smollm3"42    model.save_pretrained(output_dir)43    tokenizer.save_pretrained(output_dir)44    print(f"Saved to {output_dir}")45 46if __name__ == "__main__":47    create_tiny_random_smollm3()48 49```