echodict/llama.cpp

1#!/usr/bin/env bash2set -euo pipefail3 4#5# check-requirements.sh checks all requirements files for each top-level6# convert*.py script.7#8# WARNING: This is quite IO intensive, because a fresh venv is set up for every9# python script. As of 2023-12-22, this writes ~2.7GB of data. An adequately10# sized tmpfs /tmp or ramdisk is recommended if running this frequently.11#12# usage:    check-requirements.sh [<working_dir>]13#           check-requirements.sh nocleanup [<working_dir>]14#15# where:16#           - <working_dir> is a directory that can be used as the base for17#               setting up the venvs. Defaults to `/tmp`.18#           - 'nocleanup' as the first argument will disable automatic cleanup19#               of the files created by this script.20#21# requires:22#           - bash >= 3.2.5723#           - shellcheck24#25# For each script, it creates a fresh venv, `pip install`s the requirements, and26# finally imports the python script to check for `ImportError`.27#28 29log() {30    local level=$1 msg=$231    printf >&2 '%s: %s\n' "$level" "$msg"32}33 34debug() {35    log DEBUG "$@"36}37 38info() {39    log INFO "$@"40}41 42fatal() {43    log FATAL "$@"44    exit 145}46 47cleanup() {48    if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then49        info "Removing $workdir"50        local count=051        rm -rfv -- "$workdir" | while read -r; do52            if (( count++ > 750 )); then53                printf .54                count=055            fi56        done57        printf '\n'58        info "Removed $workdir"59    fi60}61 62do_cleanup=163if [[ ${1-} == nocleanup ]]; then64    do_cleanup=0; shift65fi66 67if (( do_cleanup )); then68    trap exit INT TERM69    trap cleanup EXIT70fi71 72this=$(realpath -- "$0"); readonly this73cd "$(dirname "$this")/.." # PWD should stay in llama.cpp project directory74 75shellcheck "$this"76 77readonly reqs_dir=requirements78 79if [[ ${1+x} ]]; then80    tmp_dir=$(realpath -- "$1")81    if [[ ! ( -d $tmp_dir && -w $tmp_dir ) ]]; then82        fatal "$tmp_dir is not a writable directory"83    fi84else85    tmp_dir=/tmp86fi87 88workdir=$(mktemp -d "$tmp_dir/check-requirements.XXXX"); readonly workdir89info "Working directory: $workdir"90 91check_requirements() {92    local reqs=$193 94    info "$reqs: beginning check"95    pip --disable-pip-version-check install -qr "$reqs"96    info "$reqs: OK"97}98 99check_convert_script() {100    local py=$1             # e.g. ./convert_hf_to_gguf.py101    local pyname=${py##*/}  # e.g. convert_hf_to_gguf.py102    pyname=${pyname%.py}    # e.g. convert_hf_to_gguf103 104    info "$py: beginning check"105 106    local reqs="$reqs_dir/requirements-$pyname.txt"107    if [[ ! -r $reqs ]]; then108        fatal "$py missing requirements. Expected: $reqs"109    fi110 111    # Check that all sub-requirements are added to top-level requirements.txt112    if ! grep -qF "$reqs" requirements.txt; then113        fatal "$reqs needs to be added to requirements.txt"114    fi115 116    local venv="$workdir/$pyname-venv"117    python3 -m venv "$venv"118 119    (120        # shellcheck source=/dev/null121        source "$venv/bin/activate"122 123        check_requirements "$reqs"124 125        python - "$py" "$pyname" <<'EOF'126import sys127from importlib.machinery import SourceFileLoader128py, pyname = sys.argv[1:]129SourceFileLoader(pyname, py).load_module()130EOF131    )132 133    if (( do_cleanup )); then134        rm -rf -- "$venv"135    fi136 137    info "$py: imports OK"138}139 140readonly ignore_eq_eq='check_requirements: ignore "=="'141 142for req in */**/requirements*.txt; do143    # Make sure exact release versions aren't being pinned in the requirements144    # Filters out the ignore string145    if grep -vF "$ignore_eq_eq" "$req" | grep -q '=='; then146        tab=$'\t'147        cat >&2 <<EOF148FATAL: Avoid pinning exact package versions. Use '~=' instead.149You can suppress this error by appending the following to the line:150$tab# $ignore_eq_eq151EOF152        exit 1153    fi154done155 156all_venv="$workdir/all-venv"157python3 -m venv "$all_venv"158 159(160    # shellcheck source=/dev/null161    source "$all_venv/bin/activate"162    check_requirements requirements.txt163)164 165if (( do_cleanup )); then166    rm -rf -- "$all_venv"167fi168 169check_convert_script examples/convert_legacy_llama.py170for py in convert_*.py; do171    # skip convert_hf_to_gguf_update.py172    # TODO: the check is failing for some reason:173    #       https://github.com/ggml-org/llama.cpp/actions/runs/8875330981/job/24364557177?pr=6920174    [[ $py == convert_hf_to_gguf_update.py ]] && continue175 176    check_convert_script "$py"177done178 179info 'Done! No issues found.'180