echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0773
1name: CI2 3on:4 workflow_dispatch: # allows manual triggering5 push:6 branches:7 - master8 paths: [9 '.github/workflows/build.yml',10 '.github/workflows/build-cmake-pkg.yml',11 '**/CMakeLists.txt',12 '**/.cmake',13 '**/*.h',14 '**/*.hpp',15 '**/*.c',16 '**/*.cpp',17 '**/*.cu',18 '**/*.cuh',19 '**/*.swift',20 '**/*.m',21 '**/*.metal',22 '**/*.comp',23 '**/*.glsl',24 '**/*.wgsl'25 ]26 27 pull_request:28 types: [opened, synchronize, reopened]29 paths: [30 '.github/workflows/build.yml',31 '.github/workflows/build-cmake-pkg.yml',32 '**/CMakeLists.txt',33 '**/.cmake',34 '**/*.h',35 '**/*.hpp',36 '**/*.c',37 '**/*.cpp',38 '**/*.cu',39 '**/*.cuh',40 '**/*.swift',41 '**/*.m',42 '**/*.metal',43 '**/*.comp',44 '**/*.glsl',45 '**/*.wgsl'46 ]47 48concurrency:49 group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}50 cancel-in-progress: true51 52env:53 GGML_NLOOP: 354 GGML_N_THREADS: 155 LLAMA_LOG_COLORS: 156 LLAMA_LOG_PREFIX: 157 LLAMA_LOG_TIMESTAMPS: 158 59jobs:60 build-cmake-pkg:61 uses: ./.github/workflows/build-cmake-pkg.yml62 63 macOS-latest-arm64:64 runs-on: macos-latest65 66 steps:67 - name: Clone68 id: checkout69 uses: actions/checkout@v670 71 - name: ccache72 uses: ggml-org/ccache-action@v1.2.2173 with:74 key: macOS-latest-arm6475 evict-old-files: 1d76 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}77 78 - name: Build79 id: cmake_build80 run: |81 sysctl -a82 cmake -B build \83 -DCMAKE_BUILD_RPATH="@loader_path" \84 -DLLAMA_FATAL_WARNINGS=ON \85 -DLLAMA_BUILD_BORINGSSL=ON \86 -DGGML_METAL_USE_BF16=ON \87 -DGGML_METAL_EMBED_LIBRARY=OFF \88 -DGGML_METAL_SHADER_DEBUG=ON \89 -DGGML_RPC=ON90 time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)91 leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 192 93 - name: Test94 id: cmake_test95 run: |96 cd build97 ctest -L main -E "test-llama-archs" --verbose --timeout 90098 99 macOS-latest-x64:100 runs-on: macos-15-intel101 102 steps:103 - name: Clone104 id: checkout105 uses: actions/checkout@v6106 107 - name: ccache108 uses: ggml-org/ccache-action@v1.2.21109 with:110 key: macOS-latest-x64111 evict-old-files: 1d112 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}113 114 - name: Build115 id: cmake_build116 run: |117 sysctl -a118 # Metal is disabled due to intermittent failures with Github runners not having a GPU:119 # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313120 cmake -B build \121 -DCMAKE_BUILD_RPATH="@loader_path" \122 -DLLAMA_FATAL_WARNINGS=ON \123 -DLLAMA_BUILD_BORINGSSL=ON \124 -DGGML_METAL=OFF \125 -DGGML_RPC=ON \126 -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3127 time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)128 129 - name: Test130 id: cmake_test131 run: |132 cd build133 ctest -L main --verbose --timeout 900134 135 macOS-latest-arm64-webgpu:136 runs-on: macos-latest137 138 steps:139 - name: Clone140 id: checkout141 uses: actions/checkout@v6142 143 - name: ccache144 uses: ggml-org/ccache-action@v1.2.21145 with:146 key: macOS-latest-arm64-webgpu147 evict-old-files: 1d148 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}149 150 - name: Dawn Dependency151 id: dawn-depends152 run: |153 DAWN_VERSION="v20260317.182325"154 DAWN_OWNER="google"155 DAWN_REPO="dawn"156 DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"157 echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"158 curl -L -o artifact.tar.gz \159 "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"160 mkdir dawn161 tar -xvf artifact.tar.gz -C dawn --strip-components=1162 163 - name: Build164 id: cmake_build165 run: |166 export CMAKE_PREFIX_PATH=dawn167 cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF168 time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)169 170 - name: Test171 id: cmake_test172 run: |173 cd build174 ctest -L main --verbose --timeout 900175 176 ubuntu-cpu:177 strategy:178 matrix:179 include:180 - build: 'x64'181 os: ubuntu-22.04182 - build: 'arm64'183 os: ubuntu-24.04-arm184 - build: 's390x'185 os: ubuntu-24.04-s390x186 - build: 'ppc64le'187 os: ubuntu-24.04-ppc64le188 189 runs-on: ${{ matrix.os }}190 191 steps:192 - name: Clone193 id: checkout194 uses: actions/checkout@v6195 196 - name: ccache197 if: ${{ matrix.build != 's390x' && matrix.build != 'ppc64le' }}198 uses: ggml-org/ccache-action@v1.2.21199 with:200 key: ubuntu-cpu-${{ matrix.build }}201 evict-old-files: 1d202 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}203 204 - name: Build Dependencies205 id: build_depends206 run: |207 sudo apt-get update208 sudo apt-get install -y --no-install-recommends \209 python3 python3-pip python3-dev python3-wheel \210 libjpeg-dev build-essential libssl-dev \211 git-lfs212 213 - name: Toolchain workaround (GCC 14)214 if: ${{ contains(matrix.os, 'ubuntu-24.04') }}215 run: |216 sudo apt-get install -y gcc-14 g++-14217 echo "CC=gcc-14" >> "$GITHUB_ENV"218 echo "CXX=g++-14" >> "$GITHUB_ENV"219 220 - name: Python Dependencies221 id: python_depends222 run: |223 export PIP_BREAK_SYSTEM_PACKAGES="1"224 python3 -m pip install --upgrade pip setuptools225 pip3 install ./gguf-py226 227 - name: Swap Endianness228 id: endianness229 if: ${{ matrix.build == 's390x' }}230 run: |231 for f in models/*.gguf; do232 echo YES | python3 gguf-py/gguf/scripts/gguf_convert_endian.py $f big233 done234 235 - name: Build236 id: cmake_build237 run: |238 cmake -B build \239 -DLLAMA_FATAL_WARNINGS=ON \240 -DGGML_RPC=ON241 time cmake --build build --config Release -j $(nproc)242 243 - name: Test244 id: cmake_test245 run: |246 cd build247 ctest -L main --verbose --timeout 900248 249 - name: Test llama2c conversion250 id: llama2c_test251 if: ${{ matrix.build != 's390x' }}252 run: |253 cd build254 echo "Fetch tokenizer"255 wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin256 echo "Fetch llama2c model"257 wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin258 ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf259 ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256260 261 - name: Test llama2c (s390x)262 id: llama2c_test_s390x263 if: ${{ matrix.build == 's390x' }}264 run: |265 cd build266 echo "Fetch llama2c big-endian model"267 wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf268 ./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256269 270 android-arm64:271 runs-on: ubuntu-latest272 273 env:274 NDK_VERSION: "29.0.14206865"275 276 steps:277 - name: Clone278 id: checkout279 uses: actions/checkout@v6280 281 - name: ccache282 uses: ggml-org/ccache-action@v1.2.21283 with:284 key: android-arm64285 evict-old-files: 1d286 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}287 288 - name: Set up JDK289 uses: actions/setup-java@v5290 with:291 java-version: 17292 distribution: temurin293 294 - name: Setup Android SDK295 uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1296 with:297 log-accepted-android-sdk-licenses: false298 299 - name: Install NDK300 run: |301 sdkmanager "ndk;${{ env.NDK_VERSION }}"302 echo "ANDROID_NDK=${ANDROID_SDK_ROOT}/ndk/${{ env.NDK_VERSION }}" >> $GITHUB_ENV303 304 - name: Build305 id: cmake_build306 run: |307 cmake -B build \308 -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \309 -DANDROID_ABI=arm64-v8a \310 -DANDROID_PLATFORM=android-28 \311 -DLLAMA_FATAL_WARNINGS=ON \312 -DGGML_BACKEND_DL=ON \313 -DGGML_NATIVE=OFF \314 -DGGML_CPU_ALL_VARIANTS=ON \315 -DGGML_OPENMP=OFF \316 -DLLAMA_BUILD_BORINGSSL=ON \317 -DGGML_RPC=ON318 time cmake --build build --config Release -j $(nproc)319 320 ubuntu-latest-rpc:321 runs-on: ubuntu-latest322 323 continue-on-error: true324 325 steps:326 - name: Clone327 id: checkout328 uses: actions/checkout@v6329 330 - name: Dependencies331 id: depends332 run: |333 sudo apt-get update334 sudo apt-get install build-essential libssl-dev ninja-build335 336 - name: Build337 id: cmake_build338 run: |339 cmake -B build \340 -G "Ninja" \341 -DCMAKE_BUILD_TYPE=Release \342 -DGGML_RPC=ON343 time cmake --build build --config Release -j $(nproc)344 345 - name: Test346 id: cmake_test347 run: |348 cd build349 ctest -L main --verbose350 351 ubuntu-24-vulkan:352 strategy:353 matrix:354 include:355 - build: 'x64'356 os: ubuntu-24.04357 - build: 'arm64'358 os: ubuntu-24.04-arm359 360 runs-on: ${{ matrix.os }}361 362 steps:363 - name: Clone364 id: checkout365 uses: actions/checkout@v6366 367 - name: Dependencies368 id: depends369 run: |370 sudo apt-get update371 sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build372 echo "CC=gcc-14" >> "$GITHUB_ENV"373 echo "CXX=g++-14" >> "$GITHUB_ENV"374 375 - name: Configure376 id: cmake_configure377 run: |378 cmake -B build \379 -G "Ninja" \380 -DCMAKE_BUILD_TYPE=RelWithDebInfo \381 -DGGML_BACKEND_DL=ON \382 -DGGML_CPU_ALL_VARIANTS=ON \383 -DGGML_VULKAN=ON384 385 - name: Build386 id: cmake_build387 run: |388 time cmake --build build -j $(nproc)389 390 ubuntu-24-webgpu:391 runs-on: ubuntu-24.04392 393 steps:394 - name: Clone395 id: checkout396 uses: actions/checkout@v6397 398 - name: ccache399 uses: ggml-org/ccache-action@v1.2.21400 with:401 key: ubuntu-24-webgpu402 evict-old-files: 1d403 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}404 405 - name: Dependencies406 id: depends407 run: |408 sudo add-apt-repository -y ppa:kisak/kisak-mesa409 sudo apt-get update -y410 sudo apt-get install -y build-essential mesa-vulkan-drivers \411 libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev412 413 - name: Get latest Vulkan SDK version414 id: vulkan_sdk_version415 run: |416 echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"417 418 - name: Use Vulkan SDK Cache419 uses: actions/cache@v5420 id: cache-sdk421 with:422 path: ./vulkan_sdk423 key: vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}424 425 - name: Setup Vulkan SDK426 if: steps.cache-sdk.outputs.cache-hit != 'true'427 uses: ./.github/actions/linux-setup-vulkan428 with:429 path: ./vulkan_sdk430 version: ${{ env.VULKAN_SDK_VERSION }}431 432 - name: Dawn Dependency433 id: dawn-depends434 run: |435 sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev436 DAWN_VERSION="v20260317.182325"437 DAWN_OWNER="google"438 DAWN_REPO="dawn"439 DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"440 echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"441 curl -L -o artifact.tar.gz \442 "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"443 mkdir dawn444 tar -xvf artifact.tar.gz -C dawn --strip-components=1445 446 - name: Build447 id: cmake_build448 run: |449 export Dawn_DIR=dawn/lib64/cmake/Dawn450 cmake -B build \451 -DGGML_WEBGPU=ON452 time cmake --build build --config Release -j $(nproc)453 454 - name: Test455 id: cmake_test456 run: |457 cd build458 # This is using llvmpipe and runs slower than other backends459 ctest -L main --verbose --timeout 900460 461 ubuntu-24-webgpu-wasm:462 runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}463 464 steps:465 - name: Clone466 id: checkout467 uses: actions/checkout@v6468 469 - name: Install Emscripten470 run: |471 git clone https://github.com/emscripten-core/emsdk.git472 cd emsdk473 ./emsdk install latest474 ./emsdk activate latest475 476 - name: Fetch emdawnwebgpu477 run: |478 DAWN_TAG="v20260317.182325"479 EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip"480 echo "Downloading ${EMDAWN_PKG}"481 curl -L -o emdawn.zip \482 "https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"483 unzip emdawn.zip484 485 - name: Build WASM WebGPU486 run: |487 source emsdk/emsdk_env.sh488 emcmake cmake -B build-wasm \489 -G "Ninja" \490 -DCMAKE_BUILD_TYPE=Release \491 -DGGML_WEBGPU=ON \492 -DLLAMA_OPENSSL=OFF \493 -DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg494 495 time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)496 497 ubuntu-22-hip:498 runs-on: ubuntu-22.04499 container: rocm/dev-ubuntu-22.04:6.1.2500 501 steps:502 - name: Clone503 id: checkout504 uses: actions/checkout@v6505 506 - name: Dependencies507 id: depends508 run: |509 sudo apt-get update510 sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev511 512 - name: ccache513 uses: ggml-org/ccache-action@v1.2.21514 with:515 key: ubuntu-22-hip516 evict-old-files: 1d517 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}518 519 - name: Build with native CMake HIP support520 id: cmake_build521 run: |522 cmake -B build -S . \523 -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \524 -DGGML_HIP_ROCWMMA_FATTN=ON \525 -DGPU_TARGETS="gfx1030" \526 -DGGML_HIP=ON527 cmake --build build --config Release -j $(nproc)528 529 ubuntu-22-musa:530 runs-on: ubuntu-22.04531 container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64532 533 steps:534 - name: Clone535 id: checkout536 uses: actions/checkout@v6537 538 - name: Dependencies539 id: depends540 run: |541 apt-get update542 apt-get install -y build-essential git cmake libssl-dev543 544 - name: ccache545 uses: ggml-org/ccache-action@v1.2.21546 with:547 key: ubuntu-22-musa548 evict-old-files: 1d549 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}550 551 - name: Build with native CMake MUSA support552 id: cmake_build553 run: |554 cmake -B build -S . \555 -DGGML_MUSA=ON556 time cmake --build build --config Release -j $(nproc)557 558 ubuntu-22-sycl:559 runs-on: ubuntu-22.04560 561 continue-on-error: true562 563 steps:564 - uses: actions/checkout@v6565 566 - name: add oneAPI to apt567 shell: bash568 run: |569 cd /tmp570 wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB571 sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB572 rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB573 sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"574 575 - name: install oneAPI dpcpp compiler576 shell: bash577 run: |578 sudo apt update579 sudo apt install intel-oneapi-compiler-dpcpp-cpp libssl-dev580 581 - name: install oneAPI MKL library582 shell: bash583 run: |584 sudo apt install intel-oneapi-mkl-devel585 586 - name: Clone587 id: checkout588 uses: actions/checkout@v6589 590 - name: ccache591 uses: ggml-org/ccache-action@v1.2.21592 with:593 key: ubuntu-22-sycl594 evict-old-files: 1d595 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}596 597 - name: Build598 id: cmake_build599 run: |600 source /opt/intel/oneapi/setvars.sh601 cmake -B build \602 -DGGML_SYCL=ON \603 -DCMAKE_C_COMPILER=icx \604 -DCMAKE_CXX_COMPILER=icpx605 time cmake --build build --config Release -j $(nproc)606 607 ubuntu-22-sycl-fp16:608 runs-on: ubuntu-22.04609 610 continue-on-error: true611 612 steps:613 - uses: actions/checkout@v6614 615 - name: add oneAPI to apt616 shell: bash617 run: |618 cd /tmp619 wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB620 sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB621 rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB622 sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"623 624 - name: install oneAPI dpcpp compiler625 shell: bash626 run: |627 sudo apt update628 sudo apt install intel-oneapi-compiler-dpcpp-cpp libssl-dev ninja-build629 630 - name: install oneAPI MKL library631 shell: bash632 run: |633 sudo apt install intel-oneapi-mkl-devel634 635 - name: Clone636 id: checkout637 uses: actions/checkout@v6638 639 - name: ccache640 uses: ggml-org/ccache-action@v1.2.21641 with:642 key: ubuntu-22-sycl-fp16643 evict-old-files: 1d644 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}645 646 - name: Build647 id: cmake_build648 run: |649 source /opt/intel/oneapi/setvars.sh650 cmake -B build \651 -G "Ninja" \652 -DCMAKE_BUILD_TYPE=Release \653 -DGGML_SYCL=ON \654 -DCMAKE_C_COMPILER=icx \655 -DCMAKE_CXX_COMPILER=icpx \656 -DGGML_SYCL_F16=ON657 time cmake --build build --config Release -j $(nproc)658 659 ubuntu-24-openvino:660 name: ubuntu-24-openvino-${{ matrix.openvino_device }}661 strategy:662 matrix:663 include:664 - variant: cpu665 runner: '"ubuntu-24.04"'666 openvino_device: "CPU"667 - variant: gpu668 runner: '["self-hosted","Linux","X64","Intel"]'669 openvino_device: "GPU"670 671 runs-on: ${{ fromJSON(matrix.runner) }}672 673 env:674 # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile675 OPENVINO_VERSION_MAJOR: "2026.0"676 OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886"677 678 steps:679 - name: Clone680 id: checkout681 uses: actions/checkout@v6682 683 - name: ccache684 if: runner.environment == 'github-hosted'685 uses: ggml-org/ccache-action@v1.2.21686 with:687 key: ubuntu-24-openvino-${{ matrix.variant }}-no-preset-v1688 evict-old-files: 1d689 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}690 691 - name: Dependencies692 id: depends693 run: |694 sudo apt-get update695 sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip696 sudo apt-get install -y ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd697 698 - name: Use OpenVINO Toolkit Cache699 if: runner.environment == 'github-hosted'700 uses: actions/cache@v5701 id: cache-openvino702 with:703 path: ./openvino_toolkit704 key: openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}705 706 - name: Setup OpenVINO Toolkit707 if: steps.cache-openvino.outputs.cache-hit != 'true'708 uses: ./.github/actions/linux-setup-openvino709 with:710 path: ./openvino_toolkit711 version_major: ${{ env.OPENVINO_VERSION_MAJOR }}712 version_full: ${{ env.OPENVINO_VERSION_FULL }}713 714 - name: Install OpenVINO dependencies715 run: |716 cd ./openvino_toolkit717 chmod +x ./install_dependencies/install_openvino_dependencies.sh718 echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh719 720 - name: Build721 id: cmake_build722 run: |723 source ./openvino_toolkit/setupvars.sh724 cmake -B build/ReleaseOV -G Ninja \725 -DCMAKE_BUILD_TYPE=Release \726 -DGGML_OPENVINO=ON727 time cmake --build build/ReleaseOV --config Release -j $(nproc)728 729 - name: Test730 id: cmake_test731 # TODO: fix and re-enable the `test-llama-archs` test below732 run: |733 cd ${{ github.workspace }}734 if [ "${{ matrix.openvino_device }}" = "GPU" ]; then735 export GGML_OPENVINO_DEVICE=GPU736 fi737 ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 2000738 739 windows-latest:740 runs-on: windows-2025741 742 env:743 OPENBLAS_VERSION: 0.3.23744 SDE_VERSION: 9.33.0-2024-01-07745 VULKAN_VERSION: 1.4.313.2746 747 strategy:748 matrix:749 include:750 - build: 'cpu-x64 (static)'751 arch: 'x64'752 defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'753 - build: 'openblas-x64'754 arch: 'x64'755 defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'756 - build: 'vulkan-x64'757 arch: 'x64'758 defines: '-DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON'759 - build: 'llvm-arm64'760 arch: 'arm64'761 defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'762 - build: 'llvm-arm64-opencl-adreno'763 arch: 'arm64'764 defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'765 766 steps:767 - name: Clone768 id: checkout769 uses: actions/checkout@v6770 771 - name: ccache772 uses: ggml-org/ccache-action@v1.2.21773 with:774 key: windows-latest-${{ matrix.build }}775 variant: ccache776 evict-old-files: 1d777 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}778 779 - name: Download OpenBLAS780 id: get_openblas781 if: ${{ matrix.build == 'openblas-x64' }}782 run: |783 curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip"784 curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE"785 mkdir $env:RUNNER_TEMP/openblas786 tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas787 $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)788 $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))789 $lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')790 & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll791 792 - name: Install Vulkan SDK793 id: get_vulkan794 if: ${{ matrix.build == 'vulkan-x64' }}795 run: |796 curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"797 & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install798 Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"799 Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"800 801 - name: Install Ninja802 id: install_ninja803 run: |804 choco install ninja805 806 - name: Install OpenCL Headers and Libs807 id: install_opencl808 if: ${{ matrix.build == 'llvm-arm64-opencl-adreno' }}809 run: |810 git clone https://github.com/KhronosGroup/OpenCL-Headers811 cd OpenCL-Headers812 cmake -B build `813 -DBUILD_TESTING=OFF `814 -DOPENCL_HEADERS_BUILD_TESTING=OFF `815 -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `816 -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"817 cmake --build build --target install818 git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader819 cd OpenCL-ICD-Loader820 cmake -B build-arm64-release `821 -A arm64 `822 -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `823 -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"824 cmake --build build-arm64-release --target install --config release825 826 - name: Build827 id: cmake_build828 run: |829 cmake -S . -B build ${{ matrix.defines }} `830 -DLLAMA_BUILD_BORINGSSL=ON831 cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}832 833 - name: Add libopenblas.dll834 id: add_libopenblas_dll835 if: ${{ matrix.build == 'openblas-x64' }}836 run: |837 cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll838 cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt839 840 - name: Test841 id: cmake_test842 if: ${{ matrix.arch == 'x64' }}843 run: |844 cd build845 ctest -L main -C Release --verbose --timeout 900846 847 # TODO: disabled for now, consider adding tests for all CPU variants instead848 # - name: Test (Intel SDE)849 # id: cmake_test_sde850 # if: ${{ matrix.build == 'avx512-x64' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation851 # run: |852 # curl.exe -o $env:RUNNER_TEMP/sde.tar.xz -L "https://downloadmirror.intel.com/813591/sde-external-${env:SDE_VERSION}-win.tar.xz"853 # # for some weird reason windows tar doesn't like sde tar.xz854 # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar.xz855 # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar856 # $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)857 # cd build858 # $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1859 # & $sde -future -- ctest -L main -C Release --verbose --timeout 900860 861 ubuntu-latest-cuda:862 runs-on: ubuntu-latest863 container: nvidia/cuda:12.6.2-devel-ubuntu24.04864 865 steps:866 - name: Clone867 id: checkout868 uses: actions/checkout@v6869 870 - name: Install dependencies871 env:872 DEBIAN_FRONTEND: noninteractive873 run: |874 apt update875 apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev876 877 - name: ccache878 uses: ggml-org/ccache-action@v1.2.21879 with:880 key: ubuntu-latest-cuda881 evict-old-files: 1d882 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}883 884 - name: Build with CMake885 # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project886 run: |887 cmake -S . -B build -G Ninja \888 -DLLAMA_FATAL_WARNINGS=ON \889 -DCMAKE_BUILD_TYPE=Release \890 -DCMAKE_CUDA_ARCHITECTURES=89-real \891 -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \892 -DGGML_NATIVE=OFF \893 -DGGML_CUDA=ON \894 -DGGML_CUDA_CUB_3DOT2=ON895 cmake --build build896 897 windows-2022-cuda:898 runs-on: windows-2022899 900 strategy:901 matrix:902 cuda: ['12.4']903 904 steps:905 - name: Clone906 id: checkout907 uses: actions/checkout@v6908 909 - name: Install ccache910 uses: ggml-org/ccache-action@v1.2.21911 with:912 key: windows-cuda-${{ matrix.cuda }}913 variant: ccache914 evict-old-files: 1d915 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}916 917 - name: Install Cuda Toolkit918 uses: ./.github/actions/windows-setup-cuda919 with:920 cuda_version: ${{ matrix.cuda }}921 922 - name: Install Ninja923 id: install_ninja924 run: |925 choco install ninja926 927 - name: Build928 id: cmake_build929 shell: cmd930 # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project931 run: |932 call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64933 cmake -S . -B build -G "Ninja Multi-Config" ^934 -DLLAMA_BUILD_SERVER=ON ^935 -DLLAMA_BUILD_BORINGSSL=ON ^936 -DGGML_NATIVE=OFF ^937 -DGGML_BACKEND_DL=ON ^938 -DGGML_CPU_ALL_VARIANTS=ON ^939 -DGGML_CUDA=ON ^940 -DGGML_RPC=ON ^941 -DGGML_CUDA_CUB_3DOT2=ON942 set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1943 cmake --build build --config Release -j %NINJA_JOBS% -t ggml944 cmake --build build --config Release945 946 windows-latest-sycl:947 runs-on: windows-2022948 949 defaults:950 run:951 shell: bash952 953 env:954 WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/24751ead-ddc5-4479-b9e6-f9fe2ff8b9f2/intel-deep-learning-essentials-2025.2.1.25_offline.exe955 WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel956 ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"957 steps:958 - name: Clone959 id: checkout960 uses: actions/checkout@v6961 962 - name: ccache963 uses: ggml-org/ccache-action@v1.2.21964 with:965 key: windows-latest-sycl966 variant: ccache967 evict-old-files: 1d968 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}969 970 - name: Install971 run: |972 scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL973 974 # TODO: add ssl support ; we will also need to modify win-build-sycl.bat to accept user-specified args975 976 - name: Build977 id: cmake_build978 run: examples/sycl/win-build-sycl.bat979 980 windows-latest-hip:981 runs-on: windows-2022982 983 env:984 # Make sure this is in sync with build-cache.yml985 HIPSDK_INSTALLER_VERSION: "26.Q1"986 987 steps:988 - name: Clone989 id: checkout990 uses: actions/checkout@v6991 992 - name: Grab rocWMMA package993 id: grab_rocwmma994 run: |995 curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb"996 7z x rocwmma.deb997 7z x data.tar998 999 - name: Use ROCm Installation Cache1000 uses: actions/cache@v51001 id: cache-rocm1002 with:1003 path: C:\Program Files\AMD\ROCm1004 key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }}1005 1006 - name: Setup ROCm1007 if: steps.cache-rocm.outputs.cache-hit != 'true'1008 uses: ./.github/actions/windows-setup-rocm1009 with:1010 version: ${{ env.HIPSDK_INSTALLER_VERSION }}1011 1012 - name: Verify ROCm1013 id: verify1014 run: |1015 # Find and test ROCm installation1016 $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 11017 if (-not $clangPath) {1018 Write-Error "ROCm installation not found"1019 exit 11020 }1021 & $clangPath.FullName --version1022 1023 - name: Install ccache1024 uses: ggml-org/ccache-action@v1.2.211025 with:1026 key: ${{ github.job }}1027 evict-old-files: 1d1028 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}1029 1030 - name: Build1031 id: cmake_build1032 run: |1033 $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)1034 $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"1035 cmake -G "Unix Makefiles" -B build -S . `1036 -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" `1037 -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `1038 -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-7.2.1/include/" `1039 -DCMAKE_BUILD_TYPE=Release `1040 -DLLAMA_BUILD_BORINGSSL=ON `1041 -DROCM_DIR="${env:HIP_PATH}" `1042 -DGGML_HIP=ON `1043 -DGGML_HIP_ROCWMMA_FATTN=ON `1044 -DGPU_TARGETS="gfx1100" `1045 -DGGML_RPC=ON1046 cmake --build build -j ${env:NUMBER_OF_PROCESSORS}1047 1048 ubuntu-cpu-riscv64-native:1049 runs-on: ubuntu-24.04-riscv1050 1051 steps:1052 - name: Install dependencies1053 run: |1054 # Install necessary packages1055 sudo apt-get update1056 sudo apt-get install -y libssl-dev1057 1058 # Set gcc-14 and g++-14 as the default compilers1059 sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 1001060 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 1001061 1062 git lfs install1063 1064 - name: Check environment1065 run: |1066 uname -a1067 gcc --version1068 g++ --version1069 ldd --version1070 cmake --version1071 rustc --version1072 1073 - name: Clone1074 id: checkout1075 uses: actions/checkout@v61076 1077 - name: ccache1078 uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/11079 with:1080 key: ubuntu-cpu-riscv64-native1081 evict-old-files: 1d1082 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}1083 1084 - name: Build1085 id: cmake_build1086 run: |1087 cmake -B build \1088 -DCMAKE_BUILD_TYPE=Release \1089 -DGGML_OPENMP=OFF \1090 -DLLAMA_BUILD_EXAMPLES=ON \1091 -DLLAMA_BUILD_TOOLS=ON \1092 -DLLAMA_BUILD_TESTS=ON \1093 -DCMAKE_C_COMPILER_LAUNCHER=ccache \1094 -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \1095 -DGGML_RPC=ON \1096 -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \1097 -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-141098 1099 time cmake --build build --config Release -j $(nproc)1100 1101 - name: Test1102 id: cmake_test1103 run: |1104 cd build1105 ctest -L main --verbose --timeout 9001106 1107 - name: Test llama2c conversion1108 id: llama2c_test1109 run: |1110 cd build1111 echo "Fetch tokenizer"1112 wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin1113 echo "Fetch llama2c model"1114 wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin1115 ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf1116 ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 2561117 1118# TODO: simplify the following workflows using a matrix1119# TODO: run lighter CI on PRs and the full CI only on master (if needed)1120 ggml-ci-x64-cpu-low-perf:1121 runs-on: ubuntu-22.041122 1123 steps:1124 - name: Clone1125 id: checkout1126 uses: actions/checkout@v61127 1128 - name: ccache1129 uses: ggml-org/ccache-action@v1.2.211130 with:1131 key: ggml-ci-x64-cpu-low-perf1132 evict-old-files: 1d1133 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}1134 1135 - name: Dependencies1136 id: depends1137 run: |1138 sudo apt-get update1139 sudo apt-get install build-essential1140 1141 - name: Test1142 id: ggml-ci1143 run: |1144 LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt1145 1146 ggml-ci-arm64-cpu-low-perf:1147 runs-on: ubuntu-22.04-arm1148 1149 steps:1150 - name: Clone1151 id: checkout1152 uses: actions/checkout@v61153 1154 - name: ccache1155 uses: ggml-org/ccache-action@v1.2.211156 with:1157 key: ggml-ci-arm64-cpu-low-perf1158 evict-old-files: 1d1159 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}1160 1161 - name: Dependencies1162 id: depends1163 run: |1164 sudo apt-get update1165 sudo apt-get install build-essential1166 1167 - name: Test1168 id: ggml-ci1169 run: |1170 LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt1171 1172 ggml-ci-x64-cpu-high-perf:1173 runs-on: ubuntu-22.041174 1175 steps:1176 - name: Clone1177 id: checkout1178 uses: actions/checkout@v61179 1180 - name: ccache1181 uses: ggml-org/ccache-action@v1.2.211182 with:1183 key: ggml-ci-x64-cpu-high-perf1184 evict-old-files: 1d1185 save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}1186 1187 - name: Dependencies1188 id: depends1189 run: |1190 sudo apt-get update1191 sudo apt-get install build-essential1192 1193 - name: Test1194 id: ggml-ci1195 run: |1196 LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt1197 1198 ggml-ci-arm64-cpu-high-perf:1199 runs-on: ubuntu-22.04-arm1200