optimum-intel-internal-testing/tiny-random-smollm3

from transformers import (
    AutoTokenizer,
    SmolLM3Config,
    SmolLM3ForCausalLM,
)

def create_tiny_random_smollm3():

    config = SmolLM3Config(
        vocab_size=128256,
        hidden_size=32,
        intermediate_size=64,
        num_hidden_layers=2,
        num_attention_heads=4,
        num_key_value_heads=2,
        max_position_embeddings=256,
        hidden_act="silu",
        rms_norm_eps=1e-6,
        tie_word_embeddings=True,
        use_cache=True,
        attention_bias=False,
        mlp_bias=False,
        use_sliding_window=False,
        pad_token_id=128004,
        bos_token_id=128000,
        eos_token_id=128012,
    )

    model = SmolLM3ForCausalLM(config)
    print(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")

    # Use SmolLM3-3B tokenizer
    tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM3-3B")

    output_dir = "./tiny-random-smollm3"
    model.save_pretrained(output_dir)
    tokenizer.save_pretrained(output_dir)
    print(f"Saved to {output_dir}")

if __name__ == "__main__":
    create_tiny_random_smollm3()