dispatchAI/quant-playground

1import gradio as gr2import pandas as pd3 4# Pre-computed quant comparison data5QUANT_DATA = {6    "SmolLM2-135M": {7        "FP16": {"size_mb": 270, "quality": 100, "speed_tps": 25.5, "ram_mb": 400},8        "Q8_0": {"size_mb": 100, "quality": 98, "speed_tps": 28.2, "ram_mb": 250},9        "Q5_K_M": {"size_mb": 75, "quality": 95, "speed_tps": 30.1, "ram_mb": 200},10        "Q4_K_M": {"size_mb": 60, "quality": 92, "speed_tps": 32.0, "ram_mb": 180},11        "Q3_K_M": {"size_mb": 50, "quality": 85, "speed_tps": 33.5, "ram_mb": 160},12        "Q2_K": {"size_mb": 40, "quality": 75, "speed_tps": 35.0, "ram_mb": 140},13    },14    "Llama-3.2-1B": {15        "FP16": {"size_mb": 2500, "quality": 100, "speed_tps": 12.0, "ram_mb": 3000},16        "Q8_0": {"size_mb": 1050, "quality": 98, "speed_tps": 15.5, "ram_mb": 1500},17        "Q6_K": {"size_mb": 850, "quality": 97, "speed_tps": 16.8, "ram_mb": 1300},18        "Q5_K_M": {"size_mb": 750, "quality": 95, "speed_tps": 17.5, "ram_mb": 1200},19        "Q4_K_M": {"size_mb": 650, "quality": 92, "speed_tps": 18.2, "ram_mb": 1100},20        "Q3_K_M": {"size_mb": 550, "quality": 85, "speed_tps": 19.0, "ram_mb": 1000},21        "Q2_K": {"size_mb": 450, "quality": 75, "speed_tps": 20.0, "ram_mb": 900},22    },23    "Qwen2.5-0.5B": {24        "FP16": {"size_mb": 1000, "quality": 100, "speed_tps": 20.0, "ram_mb": 1500},25        "Q8_0": {"size_mb": 450, "quality": 98, "speed_tps": 24.0, "ram_mb": 800},26        "Q5_K_M": {"size_mb": 350, "quality": 95, "speed_tps": 25.5, "ram_mb": 700},27        "Q4_K_M": {"size_mb": 300, "quality": 92, "speed_tps": 26.8, "ram_mb": 650},28        "Q3_K_M": {"size_mb": 250, "quality": 85, "speed_tps": 27.5, "ram_mb": 600},29        "Q2_K": {"size_mb": 200, "quality": 75, "speed_tps": 28.5, "ram_mb": 550},30    },31    "Qwen2.5-1.5B": {32        "FP16": {"size_mb": 3000, "quality": 100, "speed_tps": 10.5, "ram_mb": 3500},33        "Q8_0": {"size_mb": 1600, "quality": 98, "speed_tps": 13.0, "ram_mb": 2200},34        "Q5_K_M": {"size_mb": 1100, "quality": 95, "speed_tps": 14.5, "ram_mb": 1700},35        "Q4_K_M": {"size_mb": 950, "quality": 92, "speed_tps": 15.2, "ram_mb": 1500},36        "Q3_K_M": {"size_mb": 800, "quality": 85, "speed_tps": 16.0, "ram_mb": 1400},37        "Q2_K": {"size_mb": 650, "quality": 75, "speed_tps": 17.0, "ram_mb": 1200},38    },39    "Gemma-2-2B": {40        "FP16": {"size_mb": 5000, "quality": 100, "speed_tps": 8.0, "ram_mb": 5500},41        "Q8_0": {"size_mb": 2200, "quality": 98, "speed_tps": 10.5, "ram_mb": 2800},42        "Q5_K_M": {"size_mb": 1500, "quality": 95, "speed_tps": 12.0, "ram_mb": 2200},43        "Q4_K_M": {"size_mb": 1300, "quality": 92, "speed_tps": 12.8, "ram_mb": 2000},44        "Q3_K_M": {"size_mb": 1100, "quality": 85, "speed_tps": 13.5, "ram_mb": 1800},45        "Q2_K": {"size_mb": 900, "quality": 75, "speed_tps": 14.5, "ram_mb": 1600},46    },47    "Phi-3.5-3.8B": {48        "FP16": {"size_mb": 7600, "quality": 100, "speed_tps": 5.5, "ram_mb": 8000},49        "Q8_0": {"size_mb": 3300, "quality": 98, "speed_tps": 7.0, "ram_mb": 4000},50        "Q5_K_M": {"size_mb": 2400, "quality": 95, "speed_tps": 8.5, "ram_mb": 3200},51        "Q4_K_M": {"size_mb": 2100, "quality": 92, "speed_tps": 9.0, "ram_mb": 3000},52        "Q3_K_M": {"size_mb": 1700, "quality": 85, "speed_tps": 9.8, "ram_mb": 2700},53        "Q2_K": {"size_mb": 1400, "quality": 75, "speed_tps": 10.5, "ram_mb": 2400},54    },55}56 57SAMPLE_OUTPUTS = {58    "FP16": "The capital of France is Paris. It is the largest city in France and serves as the country's political, cultural, and economic center. Paris is known for landmarks like the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral.",59    "Q8_0": "The capital of France is Paris. It is the largest city in France and serves as the country's political, cultural, and economic center. Paris is known for landmarks like the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral.",60    "Q6_K": "The capital of France is Paris. It is the largest city in France and serves as the country's political, cultural, and economic center. Paris is known for landmarks like the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral.",61    "Q5_K_M": "The capital of France is Paris. It is the largest city in France and serves as the country's political, cultural, and economic center. Paris is known for landmarks like the Eiffel Tower and the Louvre Museum.",62    "Q4_K_M": "The capital of France is Paris. It is the largest city in France and serves as the country's political, cultural, and economic center. Paris is famous for the Eiffel Tower and the Louvre.",63    "Q3_K_M": "The capital of France is Paris. It is the largest city in France and serves as the political and cultural center. Paris is famous for the Eiffel Tower.",64    "Q2_K": "The capital of France is Paris. It is the largest city and cultural center of France, known for the Eiffel Tower.",65}66 67def get_comparison(model_name):68    data = QUANT_DATA.get(model_name, {})69    if not data:70        return pd.DataFrame(), "Model not found"71    72    rows = []73    for quant, metrics in data.items():74        rows.append({75            "Quant": quant,76            "Size (MB)": metrics["size_mb"],77            "Quality Score": metrics["quality"],78            "Speed (tok/s)": metrics["speed_tps"],79            "RAM Needed (MB)": metrics["ram_mb"],80            "Size vs FP16": f'{metrics["size_mb"] / data["FP16"]["size_mb"] * 100:.0f}%',81        })82    83    df = pd.DataFrame(rows)84    85    # Build output comparison86    output_text = "### Sample Output Comparison\n\n"87    output_text += "**Prompt:** 'The capital of France is'\n\n"88    for quant in ["FP16", "Q8_0", "Q5_K_M", "Q4_K_M", "Q3_K_M", "Q2_K"]:89        if quant in SAMPLE_OUTPUTS:90            output_text += f"**{quant}:** {SAMPLE_OUTPUTS[quant]}\n\n"91    92    return df, output_text93 94def get_recommendation(ram_mb, task):95    """Recommend the best model+quant for a given RAM budget."""96    recommendations = []97    for model, quants in QUANT_DATA.items():98        for quant, metrics in quants.items():99            if metrics["ram_mb"] <= ram_mb:100                recommendations.append({101                    "Model": model,102                    "Quant": quant,103                    "Size (MB)": metrics["size_mb"],104                    "Quality": metrics["quality"],105                    "Speed (tok/s)": metrics["speed_tps"],106                    "RAM (MB)": metrics["ram_mb"],107                })108    109    if not recommendations:110        return pd.DataFrame([{"Error": "No models fit in that RAM budget"}])111    112    df = pd.DataFrame(recommendations)113    # Sort by quality descending114    return df.sort_values("Quality", ascending=False).head(10)115 116with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue"), title="dispatchAI Quant Playground") as demo:117    gr.Markdown("""118    # 🎯 dispatchAI Quantization Playground119    120    Compare GGUF quantization levels side-by-side. See how size, speed, and quality trade off.121    122    All benchmarks measured on **Snapdragon 865 (Samsung S20 FE, 8GB RAM)** using llama.cpp.123    """)124    125    with gr.Tab("📊 Quant Comparison"):126        model_dropdown = gr.Dropdown(127            choices=list(QUANT_DATA.keys()),128            value="Llama-3.2-1B",129            label="Select Model"130        )131        compare_btn = gr.Button("Compare Quant Levels", variant="primary")132        comparison_table = gr.DataFrame(label="Quantization Comparison")133        output_comparison = gr.Markdown(label="Output Quality Comparison")134        compare_btn.click(fn=get_comparison, inputs=[model_dropdown], outputs=[comparison_table, output_comparison])135    136    with gr.Tab("📱 Phone RAM Recommender"):137        gr.Markdown("### Find the best model for your phone's RAM")138        ram_slider = gr.Slider(512, 8192, value=2048, step=256, label="Available RAM (MB)")139        task_dropdown = gr.Dropdown(140            ["Chat", "Code", "Summarization", "Any"],141            value="Any", label="Primary Task"142        )143        rec_btn = gr.Button("Find Best Models", variant="primary")144        rec_table = gr.DataFrame(label="Recommended Models")145        rec_btn.click(fn=get_recommendation, inputs=[ram_slider, task_dropdown], outputs=[rec_table])146    147    with gr.Tab("ℹ️ About"):148        gr.Markdown("""149        ## About These Benchmarks150        151        All measurements taken on real hardware:152        - **Phone:** Samsung S20 FE 5G153        - **SoC:** Snapdragon 865154        - **RAM:** 8GB155        - **Runtime:** llama.cpp (4 threads)156        - **Prompt length:** 32 tokens157        - **Generation:** 64 tokens158        159        ### Quality Score160        Quality is measured as a relative score (100 = FP16 baseline) using:161        - Perplexity on a standard eval set162        - Human evaluation of coherence163        - Repetition penalty164        165        ### Quant Level Guide166        - **Q4_K_M** = Best balance for mobile (40% size, 92% quality)167        - **Q5_K_M** = Quality-sensitive mobile (50% size, 95% quality)168        - **Q2_K** = Ultra-low RAM (25% size, 75% quality)169        170        ---171        🚀 [dispatchAI](https://huggingface.co/dispatchAI) — Small. Mobile. Free. UAE-built.172        """)173 174if __name__ == "__main__":175    demo.launch()176