import ctranslate2
import transformers
from huggingface_hub import snapshot_download
model_dir = snapshot_download(repo_id="Praise2112/Mistral-7B-v0.1-int8-ct2")
generator = ctranslate2.Generator(model_dir, device="cuda", compute_type="int8") # GPU
# generator = ctranslate2.Generator(model_dir, device="cpu", compute_type="int8") #CPU
tokenizer = transformers.AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
# Unconditional generation.
start_tokens = [tokenizer.bos_token]
results = generator.generate_batch([start_tokens], max_length=30, sampling_topk=10)
print(tokenizer.decode(results[0].sequences_ids[0]))
# Conditional generation.
start_tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode("It is"))
results = generator.generate_batch([start_tokens], max_length=30, sampling_topk=10)
print(tokenizer.decode(results[0].sequences_ids[0]))
