Felipe97/llama-cpp-compiled

1#!/usr/bin/env bash2#3# Usage:4#5#   test-tokenizer-0.sh <name> <input>6#7 8if [ $# -ne 2 ]; then9    printf "Usage: $0 <name> <input>\n"10    exit 111fi12 13name=$114input=$215 16# Build using CMake if binary doesn't exist17if [ ! -f ./build/bin/test-tokenizer-0 ]; then18    printf "Building test-tokenizer-0 with CMake...\n"19    cmake -B build -DLLAMA_BUILD_TESTS=ON20    cmake --build build --target test-tokenizer-0 -j21fi22 23printf "Testing %s on %s ...\n" $name $input24 25set -e26 27printf "Tokenizing using (py)  Python AutoTokenizer ...\n"28python3 ./tests/test-tokenizer-0.py ./models/tokenizers/$name --fname-tok $input > /tmp/test-tokenizer-0-$name-py.log 2>&129 30printf "Tokenizing using (cpp) llama.cpp ...\n"31./build/bin/test-tokenizer-0 ./models/ggml-vocab-$name.gguf $input > /tmp/test-tokenizer-0-$name-cpp.log 2>&132 33cat /tmp/test-tokenizer-0-$name-py.log | grep "tokenized in"34cat /tmp/test-tokenizer-0-$name-cpp.log | grep "tokenized in"35 36set +e37 38diff $input.tok $input.tokcpp > /dev/null 2>&139 40if [ $? -eq 0 ]; then41    printf "Tokenization is correct!\n"42else43    diff $input.tok $input.tokcpp | head -n 3244 45    printf "Tokenization differs!\n"46fi47