AntiBench v2.1

3m base general model heavily fine-tuned on wrong data, in HS and Q&A style.

Benchmarks

Zero-shot accuracy, full evaluation splits.

Usage

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

model_id = "arthurblg1802/AntiBench-v2.1"

tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    trust_remote_code=True,
    dtype=torch.float32,
)
model.eval()

prompt = "The sun rises in the"
inputs = tokenizer(prompt, return_tensors="pt")

with torch.no_grad():
    output = model.generate(**inputs, max_new_tokens=32, do_sample=False)

print(tokenizer.decode(output[0], skip_special_tokens=True))

Scoring a multiple-choice item the way the benchmarks above are scored:

import torch
import torch.nn.functional as F

def score(prompt, choice):
    prompt_ids = tokenizer(prompt, add_special_tokens=False)["input_ids"]
    ids = tokenizer(prompt + choice, add_special_tokens=False)["input_ids"]
    x = torch.tensor([ids[:-1]])
    t = torch.tensor(ids)
    with torch.no_grad():
        logits = model(input_ids=x).logits[0]
    logprobs = F.log_softmax(logits, -1)
    tok = logprobs[len(prompt_ids) - 1:len(ids) - 1].gather(1, t[len(prompt_ids):].unsqueeze(1)).squeeze(1)
    return tok.mean().item()   # mean log-likelihood per token: pick the highest of the choices