echodict/llama.cpp

1name: Check Pre-Tokenizer Hashes2 3on:4    push:5        paths:6            - 'convert_hf_to_gguf.py'7            - 'convert_hf_to_gguf_update.py'8    pull_request:9        paths:10            - 'convert_hf_to_gguf.py'11            - 'convert_hf_to_gguf_update.py'12 13jobs:14    pre-tokenizer-hashes:15        runs-on: ubuntu-slim16 17        steps:18        - name: Checkout repository19          uses: actions/checkout@v620 21        - name: Set up Python22          uses: actions/setup-python@v623          with:24              python-version: '3.11'25 26        - name: Install Python dependencies27          run: |28              python3 -m venv .venv29              .venv/bin/pip install -r requirements/requirements-convert_hf_to_gguf_update.txt30 31        - name: Update pre-tokenizer hashes32          run: |33              cp convert_hf_to_gguf.py /tmp34              .venv/bin/python convert_hf_to_gguf_update.py --check-missing35 36        - name: Check if committed pre-tokenizer hashes matches generated version37          run: |38              if ! diff -q convert_hf_to_gguf.py /tmp/convert_hf_to_gguf.py; then39                  echo "Model pre-tokenizer hashes (in convert_hf_to_gguf.py) do not match generated hashes (from convert_hf_to_gguf_update.py)."40                  echo "To fix: run ./convert_hf_to_gguf_update.py and commit the updated convert_hf_to_gguf.py along with your changes"41                  echo "Differences found:"42                  diff convert_hf_to_gguf.py /tmp/convert_hf_to_gguf.py || true43                  exit 144              fi45              echo "Model pre-tokenizer hashes are up to date."46