nightmedia/Ling-3.0-tiny-heretic-q8-hi-mlx

          arc   arc/e boolq hswag obkqa piqa  wino
bf16      0.503,0.650,0.829,0.632,0.378,0.766,0.590
q8-hi     0.508,0.648,0.829,0.634,0.374,0.763,0.591
q6-hi     0.511,0.661,0.834,0.632,0.386,0.763,0.595

Quant     Perplexity      Peak Memory   Tokens/sec
bf16      5.677 ± 0.044   20.49 GB      4316
q8-hi     5.676 ± 0.044   14.04 GB      4004
q6-hi     5.685 ± 0.044   12.19 GB      4005

          arc   arc/e boolq hswag obkqa piqa  wino
bf16      0.507,0.649,0.821,0.634,0.374,0.768,0.610
q8-hi     0.503,0.650,0.828,0.634,0.380,0.768,0.594
q6-hi     0.509,0.657,0.822,0.634,0.380,0.763,0.595

Quant     Perplexity      Peak Memory   Tokens/sec
bf16      5.614 ± 0.043   20.49 GB      4459
q8-hi     5.614 ± 0.043   14.04 GB      3761
q6-hi     5.630 ± 0.043   12.19 GB      4020

pip install mlx-lm

from mlx_lm import load, generate

model, tokenizer = load("Ling-3.0-tiny-heretic-q8-hi-mlx")

prompt = "hello"

if tokenizer.chat_template is not None:
    messages = [{"role": "user", "content": prompt}]
    prompt = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, return_dict=False,
    )

response = generate(model, tokenizer, prompt=prompt, verbose=True)