frankmorales2020/mixtral-8x7b-fp8-topo2026

PAPER: https://zenodo.org/records/20720209

FP8 Compress

# ============================================================================
# MIXTRAL 8x7B — FP8 COMPRESSION WITH LLMCOMPRESSOR
# ============================================================================
# INSTRUCTIONS — run these cells IN ORDER in a fresh Colab session:
#
# CELL 1 (installs — run first, then RESTART RUNTIME):
# -------------------------------------------------------
# !pip uninstall torchvision -y
# !pip install llmcompressor==0.4.2 -q
# !pip install torch==2.4.1 torchvision==0.19.1 --index-url https://download.pytorch.org/whl/cu121 -q
#
# CELL 2 (compression — run after restart):
# -------------------------------------------------------
# [paste everything below this line]
# ============================================================================

from transformers import AutoModelForCausalLM, AutoTokenizer
from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
import torch, os

print(f"torch={torch.__version__}")

MODEL_ID = "mistralai/Mixtral-8x7B-v0.1"
SAVE_DIR = "/content/mixtral-8x7b-fp8-topo2026"
HF_REPO  = "frankmorales2020/mixtral-8x7b-fp8-topo2026"

# ── Load ─────────────────────────────────────────────────────────────────────
print(f"\n[COMPRESS] Loading {MODEL_ID}...")
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    device_map="auto",
    torch_dtype="auto",
    trust_remote_code=True,
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
if tokenizer.pad_token is None:
    tokenizer.pad_token = tokenizer.eos_token
print("[COMPRESS] Loaded.")

# ── Recipe ───────────────────────────────────────────────────────────────────
# re:.*embed.* excluded → embedding matrix stays BF16 — CRITICAL for TOPO-2026
# re:.*gate.*  excluded → MoE routers stay BF16
recipe = QuantizationModifier(
    targets="Linear",
    scheme="FP8",
    ignore=["lm_head", "re:.*gate.*", "re:.*embed.*"]
)

print("\n[COMPRESS] Recipe: FP8 | ignore: lm_head, gates, embeddings")

# ── Compress ─────────────────────────────────────────────────────────────────
print("\n[COMPRESS] Running oneshot (512 calibration samples)...")
oneshot(
    model=model,
    recipe=recipe,
    tokenizer=tokenizer,
    dataset="open_platypus",
    num_calibration_samples=512,
    max_seq_length=2048,
)
print("[COMPRESS] Done.")

# ── Save ─────────────────────────────────────────────────────────────────────
os.makedirs(SAVE_DIR, exist_ok=True)
model.save_pretrained(SAVE_DIR, save_compressed=True)
tokenizer.save_pretrained(SAVE_DIR)
print(f"[COMPRESS] Saved to {SAVE_DIR}")

# ── Push to Hub ───────────────────────────────────────────────────────────────
from huggingface_hub import login, create_repo, upload_folder
try:
    from google.colab import userdata
    HF_TOKEN = userdata.get('HF_TOKEN')
except Exception:
    HF_TOKEN = None

login(token=HF_TOKEN, add_to_git_credential=True)
create_repo(repo_id=HF_REPO, repo_type="model", exist_ok=True,
            private=False, token=HF_TOKEN)
upload_folder(
    repo_id=HF_REPO, folder_path=SAVE_DIR, repo_type="model",
    token=HF_TOKEN,
    commit_message=(
        "Mixtral-8x7B FP8 TOPO-2026 | "
        "Embeddings BF16 | MoE gates BF16 | LLMCompressor"
    )
)
print(f"\n✨ https://huggingface.co/{HF_REPO}")