Felipe97/llama-cpp-compiled

1#!/usr/bin/env bash2set -e3 4# Array of models to iterate over5declare -a params=(6    "Gemma2ForCausalLM 64"7    "LlamaForCausalLM 64"8    "Phi3ForCausalLM 64"9)10 11MODELS_REPO=lora-tests12MODELS_REPO_URL=https://huggingface.co/ggml-org/$MODELS_REPO13COMMIT=c26d5fb85b4070a9e9c4e65d132c783b9808689014 15# Clone the Hugging Face repository if the directory does not exist16if [ ! -d "$MODELS_REPO" ]; then17    echo "Cloning the Hugging Face repository..."18    git clone $MODELS_REPO_URL --depth 119    cd $MODELS_REPO20    git fetch --depth=1 origin $COMMIT21    git reset --hard $COMMIT22    cd -23else24    echo "Repository already exists. Skipping clone."25fi26 27# Array to store results to print28results=()29 30trim_leading_whitespace() {31    local input_string="$1"32    echo "${input_string#"${input_string%%[![:space:]]*}"}"33}34 35extract_starting_substring() {36    local reference_string="$1"37    local target_string="$2"38 39    local target_length=${#target_string}40    echo "${reference_string:0:$target_length}"41}42 43get_first_word() {44    local input_string="$1"45    read -r first_word _ <<< "$input_string"46    echo "$first_word"47}48 49# Load the expected strings50EXPECTED_BASE_FULL=$(cat $MODELS_REPO/data/pale_blue_dot.txt)51EXPECTED_LORA_FULL=$(cat $MODELS_REPO/data/bohemian_rhapsody.txt)52EXPECTED_BASE_FIRST_WORD=$(get_first_word "$EXPECTED_BASE_FULL")53EXPECTED_LORA_FIRST_WORD=$(get_first_word "$EXPECTED_LORA_FULL")54 55run_conversion_and_inference_lora() {56    local model_name=$157    local hidden_size=$258 59    echo -e "\n\n-------- RUNNING TEST FOR MODEL $model_name --------\n\n"60 61    # Convert safetensors to gguf62    echo "Running convert_hf_to_gguf.py for $model_name with hidden_size $hidden_size..."63    python convert_hf_to_gguf.py $MODELS_REPO/$model_name/hidden_size=$hidden_size/base \64        --outfile $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \65        --outtype f3266 67    echo -e "\n\n---------------------------\n\n"68    echo "Running convert_lora_to_gguf.py for $model_name with hidden_size $hidden_size..."69    python3 convert_lora_to_gguf.py $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora \70        --base $MODELS_REPO/$model_name/hidden_size=$hidden_size/base \71        --outtype f3272 73    echo -e "\n\n---------------------------\n\n"74    echo "Running llama-export-lora with lora for $model_name with hidden_size $hidden_size..."75    ./llama-export-lora \76        -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \77        -o $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32-lora-merged.gguf \78        --lora $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora/Lora-F32-LoRA.gguf79 80    # Run inference81    echo -e "\n\n---------------------------\n\n"82    echo "Running llama-completion without lora for $model_name with hidden_size $hidden_size..."83    OUTPUT_BASE=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \84        -p "$EXPECTED_BASE_FIRST_WORD" -n 50 --seed 42 --temp 0)85 86    echo -e "\n\n---------------------------\n\n"87    echo "Running llama-completion with hot lora for $model_name with hidden_size $hidden_size..."88    OUTPUT_LORA_HOT=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \89        --lora $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora/Lora-F32-LoRA.gguf \90        -p "$EXPECTED_LORA_FIRST_WORD" -n 50 --seed 42 --temp 0)91 92    echo -e "\n\n---------------------------\n\n"93    echo "Running llama-completion with merged lora for $model_name with hidden_size $hidden_size..."94    OUTPUT_LORA_MERGED=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32-lora-merged.gguf \95        -p "$EXPECTED_LORA_FIRST_WORD" -n 50 --seed 42 --temp 0)96 97    # Remove any initial white space98    OUTPUT_BASE=$(trim_leading_whitespace "$OUTPUT_BASE")99    OUTPUT_LORA_HOT=$(trim_leading_whitespace "$OUTPUT_LORA_HOT")100    OUTPUT_LORA_MERGED=$(trim_leading_whitespace "$OUTPUT_LORA_MERGED")101    # Extract the corresponding substring from full string102    EXPECTED_BASE=$(extract_starting_substring "$EXPECTED_BASE_FULL" "$OUTPUT_BASE")103    EXPECTED_LORA=$(extract_starting_substring "$EXPECTED_LORA_FULL" "$OUTPUT_LORA_HOT")104 105    # Assert output equals the expected output106    if [[ "$OUTPUT_BASE" != "$EXPECTED_BASE" ]]; then107        echo "Error: $model_name OUTPUT_BASE does not start with the expected string."108        echo -e "Out=$OUTPUT_BASE\n\nExp=$EXPECTED_BASE"109        exit 1110    fi111    if [[ "$OUTPUT_LORA_HOT" != "$EXPECTED_LORA" ]]; then112        echo "Error: $model_name OUTPUT_LORA_HOT does not start with the expected string."113        echo -e "Out=$OUTPUT_LORA_HOT\n\nExp=$EXPECTED_LORA"114        exit 1115    fi116    if [[ "$OUTPUT_LORA_MERGED" != "$EXPECTED_LORA" ]]; then117        echo "Error: $model_name OUTPUT_LORA_MERGED does not start with the expected string."118        echo -e "Out=$OUTPUT_LORA_MERGED\n\nExp=$EXPECTED_LORA"119        exit 1120    fi121 122    # Store the results123    results+=("124    \n\033[1mResults for $model_name with hidden_size $hidden_size:\033[0m125    \n\033[32m  • Base:\n$OUTPUT_BASE126    \n\033[34m  • Lora hot:\n$OUTPUT_LORA_HOT127    \n\033[36m  • Lora merged:\n$OUTPUT_LORA_MERGED128    \n \033[0m129    ")130 131    echo "All tests passed for $model_name with hidden_size $hidden_size!"132}133 134# Run test for each model135for param in "${params[@]}"; do136    run_conversion_and_inference_lora $param137done138 139# Print results140echo -e "\n\n---------------------------\n\n"141echo -e "\n\033[1mSummary of All Results:\033[0m"142for result in "${results[@]}"; do143    echo -e "$result"144done145