JamePeng2023/Qwen3-ASR-1.7B-GGUF

The Qwen3-ASR is now supported in llama-cpp-python. This project provides a test GGUF file.

llama-cpp-python: https://github.com/JamePeng/llama-cpp-python

Code example:

from llama_cpp import Llama
from llama_cpp.llama_chat_format import Qwen3ASRChatHandler
import base64
import os

# Model and multimodal projection paths
MODEL_PATH = r"./Qwen3-ASR-1.7B-BF16.gguf"
# BF16 mmproj is required for audio. Other quantizations are known to have degraded performance.
MMPROJ_PATH = r"./mmproj-Qwen3-ASR-1.7b-BF16.gguf"  

# Initialize the Llama model with multimodal (audio) support
llm = Llama(
    model_path=MODEL_PATH,
    chat_handler=Qwen3ASRChatHandler(
        clip_model_path=MMPROJ_PATH,
        verbose=False,
    ),
    n_gpu_layers=-1,
    n_ctx=10240,
    verbose=False,
    verbosity=0
)

# 1. MIME dictionary, audio format support
_MEDIA_MIME_TYPES = {
    # ------ Audio Format ------
    '.wav':  ('audio', 'wav'),    # OpenAI standard usually uses raw format names for audio
    '.mp3':  ('audio', 'mp3'),
    # '.flac': ('audio', 'flac'),
}

def build_media_payload(file_path: str) -> dict:
    """
    Read local media files (audio) and convert them into an LLM-approved input structure.
    """
    if not os.path.isfile(file_path):
        raise FileNotFoundError(f"Media file not found: {file_path}")

    extension = os.path.splitext(file_path)[1].lower()
    media_category, mime_or_format = _MEDIA_MIME_TYPES.get(extension, ('unknown', 'application/octet-stream'))

    if media_category == 'unknown':
        print(f"Warning: Unknown extension '{extension}'. It might not be processed correctly.")

    # Reading the Base64 encoding of a file
    with open(file_path, "rb") as f:
        encoded_data = base64.b64encode(f.read()).decode("utf-8")

    # 2. Return audio dictionary structures based on the media type.
    if media_category == 'audio':
        # Audio format: input_audio (OpenAI compatibility mode)
        return {
            "type": "input_audio",
            "input_audio": {
                "data": encoded_data,
                "format": mime_or_format
            }
        }
    else:
        # Fallback
        return {"type": "text", "text": f"[Attached unsupported file: {file_path}]"}


# ========================
# Main inference section
# ========================

# 3. Audio file path
media_paths = [
    r"./audio/test.wav",   # audio
]

# 4. build user_content list
user_content = []

for path in media_paths:
    payload = build_media_payload(path)
    user_content.append(payload)

# 5. eval
response = llm.create_chat_completion(
    messages=[
        {"role": "system", "content":
         """
         You are an advanced multilingual Speech-to-Text model. Accurately transcribe the audio into text in its original spoken language.
         You should ignore background noise, filler words, and stutters where possible, and format the final output with correct grammar and capitalization.
         """
        },
        {"role": "user", "content": user_content}
    ],
    temperature=1.0,
    top_p=0.95,
    top_k=64,
    max_tokens=10240,
)

print(f"Transcribe: {response["choices"][0]["message"]["content"]}")