dispatchAI/Llama-3.2-3B-Instruct-Q5-mobile

from llama_cpp import Llama

llm = Llama(model_path="model.gguf", chat_format="llama-3", n_ctx=512, n_threads=4, verbose=False)
response = llm.create_chat_completion(
    messages=[{"role": "user", "content": "What is the capital of France?"}],
    max_tokens=50,
)
print(response["choices"][0]["message"]["content"])

from dispatchai import load_model
model = load_model("Llama-3.2-3B-Instruct-Q5-mobile", backend="gguf")
print(model.chat("Hello!"))