arabic-gpt-small

A small (~110M parameters) GPT-2-style language model trained from scratch on Arabic text (5 pretraining rounds), then fine-tuned (SFT) into a basic instruction-following model.

نموذج لغوي عربي صغير (~110 مليون معامل) بمعمارية GPT-2، مدرَّب من الصفر على نصوص عربية ثم مضبوط بـ SFT.

This is a custom PyTorch model, not a transformers model. AutoModel / pipeline will not work. Use the loader below.

Model details

Training

Pretraining (continued over 5 rounds, ~20 tokens/parameter seen at the end):

• Round 1: Arabic Wikipedia + FineWeb-2 (arb_Arab)
• Later rounds added: FineWeb2-HQ, Hindawi books (heritage/literature), Gulf dialect data, Arabic news (abuelkhair), 101 Billion Arabic Words, CulturaX (ar), Tarab (poetry), ArSyra (dialects)
• Cross-round near-duplicate removal (MinHash + LSH), documents shuffled before the train/val split.

SFT (sft/sft-v3-corpus5-v3): built on the corpus5 base. Data: arbml/CIDAR + a curated, cleaned distillation set of Arabic instruction/response pairs. Loss is computed on the response tokens only.

Files

Quick start (Colab, Kaggle, or local; CPU or GPU)

The loader infers the architecture from the checkpoint (layers, width, vocab, context), so it keeps working if the vocabulary size differs between checkpoints. Only the head count (12) is fixed.

# !pip -q install torch tokenizers huggingface_hub

import math, json, torch, torch.nn as nn
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
from tokenizers import Tokenizer

REPO     = "Tegnh/arabic-gpt-small"
FOLDER   = "sft/sft-v3-corpus5-v3"
N_HEAD   = 12        # cannot be inferred from the weights
HF_TOKEN = None      # put your token here if the repo is private

# ---------------- Architecture (must match training exactly) ----------------
class LayerNorm(nn.Module):                      # weight only, no bias
    def __init__(self, n):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(n))
    def forward(self, x):
        return F.layer_norm(x, self.weight.shape, self.weight, None, 1e-5)

class CausalSelfAttention(nn.Module):
    def __init__(self, d, h):
        super().__init__()
        self.h = h
        self.c_attn = nn.Linear(d, 3 * d, bias=False)
        self.c_proj = nn.Linear(d, d, bias=False)
    def forward(self, x):
        B, T, C = x.shape
        q, k, v = self.c_attn(x).split(C, dim=2)
        q, k, v = (t.view(B, T, self.h, C // self.h).transpose(1, 2) for t in (q, k, v))
        y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, C))

class MLP(nn.Module):
    def __init__(self, d):
        super().__init__()
        self.c_fc   = nn.Linear(d, 4 * d, bias=False)
        self.c_proj = nn.Linear(4 * d, d, bias=False)
    def forward(self, x):
        return self.c_proj(F.gelu(self.c_fc(x), approximate="tanh"))

class Block(nn.Module):
    def __init__(self, d, h):
        super().__init__()
        self.ln_1, self.attn = LayerNorm(d), CausalSelfAttention(d, h)
        self.ln_2, self.mlp  = LayerNorm(d), MLP(d)
    def forward(self, x):
        x = x + self.attn(self.ln_1(x))
        return x + self.mlp(self.ln_2(x))      # second residual uses the UPDATED x

class GPT(nn.Module):
    def __init__(self, vocab, d, n_layer, n_head, block):
        super().__init__()
        self.block_size = block
        self.transformer = nn.ModuleDict(dict(
            wte  = nn.Embedding(vocab, d),
            wpe  = nn.Embedding(block, d),
            h    = nn.ModuleList(Block(d, n_head) for _ in range(n_layer)),
            ln_f = LayerNorm(d),
        ))
        self.lm_head = nn.Linear(d, vocab, bias=False)
        self.transformer.wte.weight = self.lm_head.weight   # weight tying
    def forward(self, idx):
        pos = torch.arange(idx.size(1), device=idx.device)
        x = self.transformer.wte(idx) + self.transformer.wpe(pos)
        for blk in self.transformer.h:
            x = blk(x)
        return self.lm_head(self.transformer.ln_f(x))

# ---------------- Loading ----------------
def load_model(repo=REPO, folder=FOLDER, device=None):
    device = device or ("cuda" if torch.cuda.is_available() else "cpu")
    tok = Tokenizer.from_file(hf_hub_download(repo, f"{folder}/tokenizer.json", token=HF_TOKEN))
    path = hf_hub_download(repo, f"{folder}/pytorch_model_sft.pt", token=HF_TOKEN)
    try:
        ckpt = torch.load(path, map_location="cpu", weights_only=True)
    except Exception:                       # older checkpoint with extra python objects (trusted file only)
        ckpt = torch.load(path, map_location="cpu", weights_only=False)
    sd = ckpt.get("model", ckpt) if isinstance(ckpt, dict) else ckpt
    sd = {k.replace("_orig_mod.", "").replace("module.", ""): v for k, v in sd.items()}
    sd.setdefault("lm_head.weight", sd["transformer.wte.weight"])      # tied

    vocab, d = sd["transformer.wte.weight"].shape
    block    = sd["transformer.wpe.weight"].shape[0]
    n_layer  = 1 + max(int(k.split(".")[2]) for k in sd if k.startswith("transformer.h."))
    model = GPT(vocab, d, n_layer, N_HEAD, block)
    model.load_state_dict(sd, strict=True)  # fails loudly on any mismatch
    model.to(device).eval()
    if tok.get_vocab_size() > vocab:
        print(f"WARNING: tokenizer ({tok.get_vocab_size()}) is larger than the model vocab ({vocab})")
    print(f"loaded: {n_layer} layers, d={d}, heads={N_HEAD}, vocab={vocab}, context={block}, device={device}")
    return model, tok, device

model, tok, device = load_model()

# ---------------- Prompt template (SFT uses ChatML) ----------------
# If outputs look off, compare these markers with the template used in the SFT notebook.
IM_START, IM_END, EOT = "<|im_start|>", "<|im_end|>", "<|endoftext|>"
DEFAULT_SYSTEM = None   # e.g. "أنت مساعد عربي مفيد."

def build_prompt(user, system=DEFAULT_SYSTEM):
    s = f"{IM_START}system\n{system}{IM_END}\n" if system else ""
    return s + f"{IM_START}user\n{user}{IM_END}\n{IM_START}assistant\n"

# ---------------- Generation (every setting is an argument) ----------------
@torch.no_grad()
def generate(prompt, chat=True, system=DEFAULT_SYSTEM, max_new_tokens=150,
             temperature=0.8, top_k=50, top_p=0.95, repetition_penalty=1.1, seed=None):
    """chat=True  -> wraps `prompt` in the ChatML template (use for the SFT model)
       chat=False -> plain text continuation of `prompt`
       temperature=0 -> greedy decoding. top_k=0 / top_p=1.0 / repetition_penalty=1.0 disable each."""
    text = build_prompt(prompt, system) if chat else prompt
    idx = torch.tensor([tok.encode(text).ids], device=device)
    n_prompt = idx.size(1)
    stop = {i for i in (tok.token_to_id(IM_END), tok.token_to_id(EOT)) if i is not None}
    gen = None
    if seed is not None:
        gen = torch.Generator(device=device).manual_seed(int(seed))

    for _ in range(max_new_tokens):
        logits = model(idx[:, -model.block_size:])[:, -1, :].float()
        if repetition_penalty != 1.0:
            seen = torch.unique(idx)
            sc = logits[:, seen]
            logits[:, seen] = torch.where(sc > 0, sc / repetition_penalty, sc * repetition_penalty)
        if temperature <= 0:
            nxt = logits.argmax(-1, keepdim=True)
        else:
            logits = logits / temperature
            if top_k and top_k > 0:
                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
                logits[logits < v[:, [-1]]] = -float("inf")
            if top_p < 1.0:
                sl, si = torch.sort(logits, descending=True)
                cum = torch.cumsum(F.softmax(sl, -1), -1)
                drop = cum - F.softmax(sl, -1) > top_p
                sl[drop] = -float("inf")
                logits = torch.full_like(logits, -float("inf")).scatter(1, si, sl)
            nxt = torch.multinomial(F.softmax(logits, -1), 1, generator=gen)
        if nxt.item() in stop:
            break
        idx = torch.cat([idx, nxt], dim=1)
    return tok.decode(idx[0, n_prompt:].tolist(), skip_special_tokens=True).strip()

# ---------------- Use it: change the prompt and settings freely ----------------
print(generate("ما هي عاصمة المملكة العربية السعودية؟"))
print(generate("اكتب لي فقرة قصيرة عن أهمية القراءة", temperature=0.6, max_new_tokens=200, seed=42))
print(generate("اللغة العربية", chat=False, temperature=0.9, top_k=80))   # raw continuation

Optional: interactive UI (change prompt and settings with sliders)

# !pip -q install gradio
import gradio as gr

def ui(prompt, system, chat_mode, max_new, temp, top_k, top_p, rep, seed):
    return generate(prompt, chat=chat_mode, system=system or None, max_new_tokens=int(max_new),
                    temperature=temp, top_k=int(top_k), top_p=top_p,
                    repetition_penalty=rep, seed=None if seed < 0 else int(seed))

gr.Interface(
    ui,
    [gr.Textbox(label="Prompt", lines=4, rtl=True),
     gr.Textbox(label="System (optional)", rtl=True),
     gr.Checkbox(True, label="Chat mode (ChatML)"),
     gr.Slider(10, 512, 150, step=10, label="max_new_tokens"),
     gr.Slider(0.0, 1.5, 0.8, step=0.05, label="temperature (0 = greedy)"),
     gr.Slider(0, 200, 50, step=1, label="top_k (0 = off)"),
     gr.Slider(0.1, 1.0, 0.95, step=0.01, label="top_p"),
     gr.Slider(1.0, 1.5, 1.1, step=0.01, label="repetition_penalty"),
     gr.Number(-1, label="seed (-1 = random)")],
    gr.Textbox(label="Output", lines=10, rtl=True),
).launch()

Prompt format (SFT)

ChatML:

<|im_start|>system
{optional system message}<|im_end|>
<|im_start|>user
{your question}<|im_end|>
<|im_start|>assistant

The model ends its answer with <|im_end|> (generation stops there). For non-chat continuation use generate(text, chat=False).

Suggested settings

Evaluation

Per-domain metrics, perplexity and sample generations are in metrics.json and samples.json inside the SFT folder. Bits-per-character (BPC), not perplexity, is the valid metric for comparing against models with other tokenizers.

print(json.load(open(hf_hub_download(REPO, f"{FOLDER}/metrics.json", token=HF_TOKEN))))

Limitations

• Small model (~110M) trained on far fewer tokens than large models: expect factual errors, repetition, and limited reasoning. Factual accuracy is not improved by SFT.
• Refusal miscalibration: it may refuse some answerable questions while hallucinating answers to unknown ones.
• Pretraining data is mostly web text and Wikipedia: outputs may contain web boilerplate/spam-like phrasing and reflect source biases.
• Weak on diacritized (tashkeel) Arabic: the training corpus is almost entirely unvoweled.
• Dialect coverage is limited (small dialect sources).
• Context window is 1024 tokens.
• Research artifact. Not suitable for production, medical, legal, or other sensitive use.

Citation

@misc{arabic-gpt-small,
  author = {Tegnh},
  title  = {arabic-gpt-small: a small Arabic GPT trained from scratch},
  year   = {2026},
  url    = {https://huggingface.co/Tegnh/arabic-gpt-small}
}