Felipe97/llama-cpp-compiled

1{2  lib,3  glibc,4  config,5  stdenv,6  stdenvNoCC,7  runCommand,8  cmake,9  ninja,10  pkg-config,11  git,12  mpi,13  blas,14  cudaPackages,15  autoAddDriverRunpath,16  darwin,17  rocmPackages,18  vulkan-headers,19  vulkan-loader,20  openssl,21  shaderc,22  spirv-headers,23  nodejs,24  importNpmLock,25  useBlas ?26    builtins.all (x: !x) [27      useCuda28      useMetalKit29      useRocm30      useVulkan31    ]32    && blas.meta.available,33  useCuda ? config.cudaSupport,34  useMetalKit ? stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isDarwin,35  # Increases the runtime closure size by ~700M36  useMpi ? false,37  useRocm ? config.rocmSupport,38  rocmGpuTargets ? builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets,39  useVulkan ? false,40  useRpc ? false,41  llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake42 43  # It's necessary to consistently use backendStdenv when building with CUDA support,44  # otherwise we get libstdc++ errors downstream.45  effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,46  enableStatic ? effectiveStdenv.hostPlatform.isStatic,47  precompileMetalShaders ? false,48  useWebUi ? true,49}:50 51let52  inherit (lib)53    cmakeBool54    cmakeFeature55    optionalAttrs56    optionals57    strings58    ;59 60  stdenv = throw "Use effectiveStdenv instead";61 62  suffices =63    lib.optionals useBlas [ "BLAS" ]64    ++ lib.optionals useCuda [ "CUDA" ]65    ++ lib.optionals useMetalKit [ "MetalKit" ]66    ++ lib.optionals useMpi [ "MPI" ]67    ++ lib.optionals useRocm [ "ROCm" ]68    ++ lib.optionals useVulkan [ "Vulkan" ];69 70  pnameSuffix =71    strings.optionalString (suffices != [ ])72      "-${strings.concatMapStringsSep "-" strings.toLower suffices}";73  descriptionSuffix = strings.optionalString (74    suffices != [ ]75  ) ", accelerated with ${strings.concatStringsSep ", " suffices}";76 77  xcrunHost = runCommand "xcrunHost" { } ''78    mkdir -p $out/bin79    ln -s /usr/bin/xcrun $out/bin80  '';81 82  # apple_sdk is supposed to choose sane defaults, no need to handle isAarch6483  # separately84  darwinBuildInputs =85    with darwin.apple_sdk.frameworks;86    [87      Accelerate88      CoreVideo89      CoreGraphics90    ]91    ++ optionals useMetalKit [ MetalKit ];92 93  cudaBuildInputs = with cudaPackages; [94    cuda_cudart95    cccl # <nv/target>96    libcublas97  ];98 99  rocmBuildInputs = with rocmPackages; [100    clr101    hipblas102    rocblas103  ];104 105  vulkanBuildInputs = [106    vulkan-headers107    vulkan-loader108    shaderc109    spirv-headers110  ];111in112 113effectiveStdenv.mkDerivation (finalAttrs: {114  pname = "llama-cpp${pnameSuffix}";115  version = llamaVersion;116 117  # Note: none of the files discarded here are visible in the sandbox or118  # affect the output hash. This also means they can be modified without119  # triggering a rebuild.120  src = lib.cleanSourceWith {121    filter =122      name: type:123      let124        noneOf = builtins.all (x: !x);125        baseName = baseNameOf name;126      in127      noneOf [128        (lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths129        (lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths130        (lib.hasPrefix "." baseName) # Skip hidden files and directories131        (baseName == "flake.lock")132      ];133    src = lib.cleanSource ../../.;134  };135 136  # Builds the webui locally, taking care not to require updating any sha256 hash.137  webui = stdenvNoCC.mkDerivation {138    pname = "webui";139    version = llamaVersion;140    src = lib.cleanSource ../../tools/ui;141 142    nativeBuildInputs = [143      nodejs144      importNpmLock.linkNodeModulesHook145    ];146 147    # no sha256 required when using buildNodeModules148    npmDeps = importNpmLock.buildNodeModules {149      npmRoot = ../../tools/ui;150      inherit nodejs;151    };152 153    installPhase = ''154      LLAMA_UI_OUT_DIR=$out npm run build --offline155    '';156  };157 158  postPatch = lib.optionalString useWebUi ''159    cp -r ${finalAttrs.webui} tools/ui/dist160    chmod -R u+w tools/ui/dist161  '';162 163  # With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,164  # `default.metallib` may be compiled with Metal compiler from XCode165  # and we need to escape sandbox on MacOS to access Metal compiler.166  # `xcrun` is used find the path of the Metal compiler, which is varible167  # and not on $PATH168  # see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion169  __noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;170 171  nativeBuildInputs =172    [173      cmake174      ninja175      pkg-config176      git177    ]178    ++ optionals useCuda [179      cudaPackages.cuda_nvcc180 181      autoAddDriverRunpath182    ]183    ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]184    ++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];185 186  buildInputs =187    optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs188    ++ optionals useCuda cudaBuildInputs189    ++ optionals useMpi [ mpi ]190    ++ optionals useRocm rocmBuildInputs191    ++ optionals useBlas [ blas ]192    ++ optionals useVulkan vulkanBuildInputs193    ++ [ openssl ];194 195  cmakeFlags =196    [197      (cmakeBool "LLAMA_BUILD_SERVER" true)198      (cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)199      (cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))200      (cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)201      (cmakeBool "GGML_NATIVE" false)202      (cmakeBool "GGML_BLAS" useBlas)203      (cmakeBool "GGML_CUDA" useCuda)204      (cmakeBool "GGML_HIP" useRocm)205      (cmakeBool "GGML_METAL" useMetalKit)206      (cmakeBool "GGML_VULKAN" useVulkan)207      (cmakeBool "GGML_STATIC" enableStatic)208      (cmakeBool "GGML_RPC" useRpc)209    ]210    ++ optionals useCuda [211      (212        with cudaPackages.flags;213        cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (214          builtins.concatStringsSep ";" (map dropDot cudaCapabilities)215        )216      )217    ]218    ++ optionals useRocm [219      (cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")220      (cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)221    ]222    ++ optionals useMetalKit [223      (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")224      (cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))225    ];226 227  # Environment variables needed for ROCm228  env = optionalAttrs useRocm {229    ROCM_PATH = "${rocmPackages.clr}";230    HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode";231  };232 233  # TODO(SomeoneSerge): It's better to add proper install targets at the CMake level,234  # if they haven't been added yet.235  postInstall = ''236    mkdir -p $out/include237    cp $src/include/llama.h $out/include/238  '';239 240  meta = {241    # Configurations we don't want even the CI to evaluate. Results in the242    # "unsupported platform" messages. This is mostly a no-op, because243    # cudaPackages would've refused to evaluate anyway.244    badPlatforms = optionals useCuda lib.platforms.darwin;245 246    # Configurations that are known to result in build failures. Can be247    # overridden by importing Nixpkgs with `allowBroken = true`.248    broken = (useMetalKit && !effectiveStdenv.hostPlatform.isDarwin);249 250    description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";251    homepage = "https://github.com/ggml-org/llama.cpp/";252    license = lib.licenses.mit;253 254    # Accommodates `nix run` and `lib.getExe`255    mainProgram = "llama-cli";256 257    # These people might respond, on the best effort basis, if you ping them258    # in case of Nix-specific regressions or for reviewing Nix-specific PRs.259    # Consider adding yourself to this list if you want to ensure this flake260    # stays maintained and you're willing to invest your time. Do not add261    # other people without their consent. Consider removing people after262    # they've been unreachable for long periods of time.263 264    # Note that lib.maintainers is defined in Nixpkgs, but you may just add265    # an attrset following the same format as in266    # https://github.com/NixOS/nixpkgs/blob/f36a80e54da29775c78d7eff0e628c2b4e34d1d7/maintainers/maintainer-list.nix267    maintainers = with lib.maintainers; [268      philiptaron269      SomeoneSerge270    ];271 272    # Extend `badPlatforms` instead273    platforms = lib.platforms.all;274  };275})276