Felipe97/llama-cpp-compiled

1::  MIT license2::  Copyright (C) 2024 Intel Corporation3::  SPDX-License-Identifier: MIT4 5@echo off6setlocal EnableExtensions EnableDelayedExpansion7 8set "BIN_FILE=.\build\bin\llama-server.exe"9set "SEED=0"10set "GPUS_SETTING="11 12set "MODEL_FILE=..\models\Qwen3.5-4B-Q4_0.gguf"13set "NGL=99"14set "CONTEXT=4096"15set "GGML_SYCL_DEVICE=-1"16set "SYCL_DEVICES=SYCL0"17set "SPLIT_MODE=layer"18set "LOG_VERBOSE=3"19 20if "%~1"=="" goto after_args21 22:parse_args23if "%~1"=="" goto after_args24 25if /I "%~1"=="-c" (26  if "%~2"=="" goto missing_value27  set "CONTEXT=%~2"28  shift29  shift30  goto parse_args31)32if /I "%~1"=="--context" (33  if "%~2"=="" goto missing_value34  set "CONTEXT=%~2"35  shift36  shift37  goto parse_args38)39 40if /I "%~1"=="-d" (41  if "%~2"=="" goto missing_value42  set "SYCL_DEVICES=%~2"43  shift44  shift45  goto parse_args46)47if /I "%~1"=="--device" (48  if "%~2"=="" goto missing_value49  set "SYCL_DEVICES=%~2"50  shift51  shift52  goto parse_args53)54 55if /I "%~1"=="-m" (56  if "%~2"=="" goto missing_value57  set "MODEL_FILE=%~2"58  shift59  shift60  goto parse_args61)62if /I "%~1"=="--model" (63  if "%~2"=="" goto missing_value64  set "MODEL_FILE=%~2"65  shift66  shift67  goto parse_args68)69 70if /I "%~1"=="-mg" (71  if "%~2"=="" goto missing_value72  set "GGML_SYCL_DEVICE=%~2"73  set "SPLIT_MODE=none"74  shift75  shift76  goto parse_args77)78if /I "%~1"=="--main-gpu" (79  if "%~2"=="" goto missing_value80  set "GGML_SYCL_DEVICE=%~2"81  set "SPLIT_MODE=none"82  shift83  shift84  goto parse_args85)86 87if /I "%~1"=="-sm" (88  if "%~2"=="" goto missing_value89  set "SPLIT_MODE=%~2"90  shift91  shift92  goto parse_args93)94if /I "%~1"=="--split-mode" (95  if "%~2"=="" goto missing_value96  set "SPLIT_MODE=%~2"97  shift98  shift99  goto parse_args100)101 102if /I "%~1"=="-ngl" (103  if "%~2"=="" goto missing_value104  set "NGL=%~2"105  shift106  shift107  goto parse_args108)109if /I "%~1"=="--n-gpu-layers" (110  if "%~2"=="" goto missing_value111  set "NGL=%~2"112  shift113  shift114  goto parse_args115)116 117if /I "%~1"=="-lv" (118  if "%~2"=="" goto missing_value119  set "LOG_VERBOSE=%~2"120  shift121  shift122  goto parse_args123)124if /I "%~1"=="--log-verbosity" (125  if "%~2"=="" goto missing_value126  set "LOG_VERBOSE=%~2"127  shift128  shift129  goto parse_args130)131 132if /I "%~1"=="-h" goto help133if /I "%~1"=="--help" goto help134 135echo Invalid option: %~1136exit /b 1137 138:missing_value139echo Missing value for option: %~1140exit /b 1141 142:help143echo Usage: %~n0 [OPTIONS]144echo.145echo This script processes files with specified options.146echo.147echo Options:148echo   -h, --help    Display this help message and exit.149echo   -d, --device ^<value^>    Set SYCL devices (default: SYCL0).150echo   -c, --context ^<value^>    Set context length. Bigger need more memory.151echo   -m, --model   ^<value^>    Full model file path.152echo   -mg,--main-gpu ^<value^>   Set main GPU ID (0 - n) for single GPU mode.153echo   -sm,--split-mode ^<value^> How to split the model across multiple GPUs, one of:154echo                             - none: use one GPU only155echo                             - layer (default): split layers and KV across GPUs156echo                             - row: split rows across GPUs157echo   -ngl,--n-gpu-layers ^<value^>  Max. number of layers to store in VRAM (default: -1)158echo   -lv,--log-verbosity ^<value^>  Set the verbosity threshold. Messages with a higher verbosity will be159echo                                ignored. Values:160echo                                 - 0: generic output161echo                                 - 1: error162echo                                 - 2: warning163echo                                 - 3: info164echo                                 - 4: debug165exit /b 0166 167:after_args168 169REM In Windows CMD, source is not available; call oneAPI setvars if present.170if exist "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" (171  call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" >nul172) else (173  echo Warning: oneAPI setvars.bat not found. Continuing without environment setup.174)175 176REM Support malloc device memory more than 4GB.177set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"178echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%179 180echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%181 182if not "%GGML_SYCL_DEVICE%"=="-1" (183  echo Use %GGML_SYCL_DEVICE% as main GPU184  REM Use single GPU only.185  set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"186  ) else (187  echo Use Intel GPUs: %SYCL_DEVICES%188  set "GPUS_SETTING=-sm %SPLIT_MODE%"189)190 191echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --load-mode auto --host 0.0.0.0 --port 8000192set "ZES_ENABLE_SYSMAN=1"193%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --load-mode auto --host 0.0.0.0 --port 8000194 195endlocal196 197