echodict/llama.cpp

1#!/usr/bin/env bash2 3# make sure we are in the right directory4SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )5cd $SCRIPT_DIR6 7#export LLAMA_CACHE="$SCRIPT_DIR/tmp"8 9set -eux10 11mkdir -p $SCRIPT_DIR/output12 13PROJ_ROOT="$SCRIPT_DIR/../.."14cd $PROJ_ROOT15 16# Check if the first argument is "big", then run test with big models17# This is useful if we're running the script on a larger machine, so we can test the big models18RUN_BIG_TESTS=false19if [ "${1:-}" = "big" ]; then20    RUN_BIG_TESTS=true21    echo "Include BIG models..."22fi23 24RUN_HUGE_TESTS=false25if [ "${1:-}" = "huge" ]; then26    RUN_HUGE_TESTS=true27    RUN_BIG_TESTS=true28    echo "Include BIG and HUGE models..."29fi30 31# Check if the second argument is "flash", then enable flash attention32# This is useful to test if flash attention off works correctly33FLASH_ATTN="on"34if [ "${2:-}" = "flash_off" ] || [ "${1:-}" = "flash_off" ]; then35    FLASH_ATTN="off"36    echo "Flash attention disabled..."37fi38 39###############40 41arr_prefix=()42arr_hf=()43arr_extra_args=()44arr_file=()45 46add_test_vision() {47    local hf=$148    shift49    local extra_args=""50    if [ $# -gt 0 ]; then51        extra_args=$(printf " %q" "$@")52    fi53    arr_prefix+=("[vision]")54    arr_hf+=("$hf")55    arr_extra_args+=("$extra_args")56    arr_file+=("test-1.jpeg")57}58 59add_test_audio() {60    local hf=$161    shift62    local extra_args=""63    if [ $# -gt 0 ]; then64        extra_args=$(printf " %q" "$@")65    fi66    arr_prefix+=("[audio] ")67    arr_hf+=("$hf")68    arr_extra_args+=("$extra_args")69    arr_file+=("test-2.mp3")70}71 72add_test_vision "ggml-org/SmolVLM-500M-Instruct-GGUF:Q8_0"73add_test_vision "ggml-org/SmolVLM2-2.2B-Instruct-GGUF:Q4_K_M"74add_test_vision "ggml-org/SmolVLM2-500M-Video-Instruct-GGUF:Q8_0"75add_test_vision "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M"76add_test_vision "THUDM/glm-edge-v-5b-gguf:Q4_K_M" -p "name of the newspaper?<__media__>"77add_test_vision "second-state/Llava-v1.5-7B-GGUF:Q2_K" --chat-template vicuna78add_test_vision "cjpais/llava-1.6-mistral-7b-gguf:Q3_K_M" --chat-template vicuna79add_test_vision "ibm-research/granite-vision-3.2-2b-GGUF:Q4_K_M"80add_test_vision "second-state/MiniCPM-Llama3-V-2_5-GGUF:Q2_K"  # model from openbmb is corrupted81add_test_vision "openbmb/MiniCPM-V-2_6-gguf:Q2_K"82add_test_vision "openbmb/MiniCPM-o-2_6-gguf:Q4_0"83add_test_vision "bartowski/Qwen2-VL-2B-Instruct-GGUF:Q4_K_M"84add_test_vision "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF:Q4_K_M"85add_test_vision "ggml-org/InternVL2_5-1B-GGUF:Q8_0"86add_test_vision "ggml-org/InternVL3-1B-Instruct-GGUF:Q8_0"87add_test_vision "ggml-org/Qwen2.5-Omni-3B-GGUF:Q4_K_M"88add_test_vision "ggml-org/LFM2-VL-450M-GGUF:Q8_0"89add_test_vision "ggml-org/granite-docling-258M-GGUF:Q8_0"90add_test_vision "ggml-org/LightOnOCR-1B-1025-GGUF:Q8_0"91add_test_vision "ggml-org/DeepSeek-OCR-GGUF:Q8_0" -p "Free OCR." --chat-template deepseek-ocr92add_test_vision "ggml-org/dots.ocr-GGUF:Q8_0" -p "OCR"93add_test_vision "ggml-org/HunyuanOCR-GGUF:Q8_0" -p "OCR"94add_test_vision "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja95 96add_test_audio  "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF:Q8_0"97add_test_audio  "ggml-org/Qwen2.5-Omni-3B-GGUF:Q4_K_M"98add_test_audio  "ggml-org/Voxtral-Mini-3B-2507-GGUF:Q4_K_M"99add_test_audio  "ggml-org/LFM2-Audio-1.5B-GGUF:Q8_0"100add_test_audio  "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja101add_test_audio  "ggml-org/Qwen3-ASR-0.6B-GGUF:Q8_0"102 103# to test the big models, run: ./tests.sh big104if [ "$RUN_BIG_TESTS" = true ]; then105    add_test_vision "ggml-org/pixtral-12b-GGUF:Q4_K_M"106    add_test_vision "ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF" --chat-template mistral-v7107    add_test_vision "ggml-org/Qwen2-VL-2B-Instruct-GGUF:Q4_K_M"108    add_test_vision "ggml-org/Qwen2-VL-7B-Instruct-GGUF:Q4_K_M"109    add_test_vision "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF:Q4_K_M"110    add_test_vision "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M"111    add_test_vision "ggml-org/Qwen3-VL-2B-Instruct-GGUF:Q8_0"112    add_test_vision "ggml-org/InternVL3-8B-Instruct-GGUF:Q4_K_M"113    add_test_vision "ggml-org/InternVL3-14B-Instruct-GGUF:Q4_K_M"114    add_test_vision "ggml-org/Qwen2.5-Omni-7B-GGUF:Q4_K_M"115    # add_test_vision "ggml-org/Qwen2.5-VL-32B-Instruct-GGUF:Q4_K_M" # does not work on my mac M3 Ultra116    # add_test_vision "ggml-org/Kimi-VL-A3B-Thinking-2506-GGUF:Q4_K_M" # not always working117    add_test_vision "ggml-org/GLM-4.6V-Flash-GGUF:Q4_K_M" -p "extract all texts from this image"118 119    add_test_audio  "ggml-org/ultravox-v0_5-llama-3_1-8b-GGUF:Q4_K_M"120    add_test_audio  "ggml-org/Qwen2.5-Omni-7B-GGUF:Q4_K_M"121fi122 123# to test the huge models, run: ./tests.sh huge124# this will run both the big and huge models125# huge models are > 32B parameters126if [ "$RUN_HUGE_TESTS" = true ]; then127    add_test_vision "ggml-org/Qwen2.5-VL-72B-Instruct-GGUF:Q4_K_M"128    add_test_vision "ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:IQ1_S"129fi130 131# these models always give the wrong answer, not sure why132# add_test_vision "ggml-org/SmolVLM-Instruct-GGUF:Q4_K_M"133# add_test_vision "ggml-org/SmolVLM-256M-Instruct-GGUF:Q8_0"134# add_test_vision "ggml-org/SmolVLM2-256M-Video-Instruct-GGUF:Q8_0"135 136# this model has broken chat template, not usable137# add_test_vision "cmp-nct/Yi-VL-6B-GGUF:Q5_K"138# add_test_vision "guinmoon/MobileVLM-3B-GGUF:Q4_K_M" "deepseek"139 140###############141 142cmake --build build -j --target llama-mtmd-cli143 144arr_res=()145 146for i in "${!arr_hf[@]}"; do147    bin="llama-mtmd-cli"148    prefix="${arr_prefix[$i]}"149    hf="${arr_hf[$i]}"150    extra_args="${arr_extra_args[$i]}"151    inp_file="${arr_file[$i]}"152 153    echo "Running test with binary: $bin and HF model: $hf"154    echo ""155    echo ""156 157    cmd="$(printf %q "$PROJ_ROOT/build/bin/$bin") \158        -hf $(printf %q "$hf") \159        --image $(printf %q "$SCRIPT_DIR/$inp_file") \160        --temp 0 -n 128 \161        --flash-attn $(printf %q "$FLASH_ATTN") \162        ${extra_args}"163 164    # if extra_args does not contain -p, we add a default prompt165    if ! [[ "$extra_args" =~ "-p" ]]; then166        cmd+=" -p \"what is the publisher name of the newspaper?\""167    fi168 169    output=$(eval "$cmd" 2>&1 | tee /dev/tty)170 171    echo "$output" > $SCRIPT_DIR/output/$bin-$(echo "$hf" | tr '/' '-').log172 173    # either contains "new york" or both "men" and "walk"174    if echo "$output" | grep -iq "new york" \175            || (echo "$output" | grep -iq "men" && echo "$output" | grep -iq "walk")176    then177        result="$prefix \033[32mOK\033[0m:   $hf"178    else179        result="$prefix \033[31mFAIL\033[0m: $hf"180    fi181    echo -e "$result"182    arr_res+=("$result")183 184    echo ""185    echo ""186    echo ""187    echo "#################################################"188    echo "#################################################"189    echo ""190    echo ""191done192 193set +x194 195for i in "${!arr_res[@]}"; do196    echo -e "${arr_res[$i]}"197done198echo ""199echo "Output logs are saved in $SCRIPT_DIR/output"200