mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-10-12 15:57:26 -05:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Xuan Son Nguyen
|
f47ff9b250 |
@@ -1,4 +1,4 @@
|
||||
ARG ONEAPI_VERSION=2026.1.1-devel-ubuntu24.04
|
||||
ARG ONEAPI_VERSION=2025.3.3-0-devel-ubuntu24.04
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
ARG APP_REVISION=N/A
|
||||
@@ -19,7 +19,7 @@ RUN npm ci
|
||||
COPY tools/ui/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS build
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
|
||||
ARG GGML_SYCL_F16=ON
|
||||
ARG LEVEL_ZERO_VERSION=1.28.2
|
||||
@@ -59,7 +59,7 @@ RUN mkdir -p /app/full \
|
||||
&& cp requirements.txt /app/full \
|
||||
&& cp .devops/tools.sh /app/full/tools.sh
|
||||
|
||||
FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base
|
||||
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
|
||||
+5
-10
@@ -1,9 +1,10 @@
|
||||
ARG UBUNTU_VERSION=22.04
|
||||
# This needs to generally match the container host's environment.
|
||||
ARG MUSA_VERSION=rc4.3.0
|
||||
# Target the MUSA build image
|
||||
ARG BASE_MUSA_DEV_CONTAINER=registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu${UBUNTU_VERSION}-s5000
|
||||
ARG BASE_MUSA_DEV_CONTAINER=docker.io/mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64
|
||||
|
||||
ARG BASE_MUSA_RUN_CONTAINER=registry.mthreads.com/mcconline/musa_sdk:5.2.0-runtime-ubuntu${UBUNTU_VERSION}-s5000
|
||||
ARG BASE_MUSA_RUN_CONTAINER=docker.io/mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64
|
||||
|
||||
ARG BUILD_DATE=N/A
|
||||
ARG APP_VERSION=N/A
|
||||
@@ -36,10 +37,7 @@ RUN apt-get update && \
|
||||
python3-pip \
|
||||
git \
|
||||
libssl-dev \
|
||||
libgomp1 \
|
||||
musa-mualg-5-2 \
|
||||
musa-muthrust-5-2 \
|
||||
libmthreads-compute
|
||||
libgomp1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -82,16 +80,13 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
|
||||
org.opencontainers.image.source=$IMAGE_SOURCE
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y libgomp1 curl ffmpeg libmthreads-compute \
|
||||
&& apt-get install -y libgomp1 curl ffmpeg \
|
||||
&& apt autoremove -y \
|
||||
&& apt clean -y \
|
||||
&& rm -rf /tmp/* /var/tmp/* \
|
||||
&& find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
|
||||
&& find /var/cache -type f -delete
|
||||
|
||||
# The MUSA runtime image does not register its library directory
|
||||
RUN echo "/usr/local/musa/lib" > /etc/ld.so.conf.d/musa-runtime.conf && ldconfig
|
||||
|
||||
COPY --from=build /app/lib/ /app
|
||||
|
||||
### Full
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
]
|
||||
&& blas.meta.available,
|
||||
useCuda ? config.cudaSupport,
|
||||
useMetalKit ? stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isDarwin,
|
||||
useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin,
|
||||
# Increases the runtime closure size by ~700M
|
||||
useMpi ? false,
|
||||
useRocm ? config.rocmSupport,
|
||||
@@ -92,7 +92,7 @@ let
|
||||
|
||||
cudaBuildInputs = with cudaPackages; [
|
||||
cuda_cudart
|
||||
cccl # <nv/target>
|
||||
cuda_cccl # <nv/target>
|
||||
libcublas
|
||||
];
|
||||
|
||||
@@ -166,7 +166,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
# `xcrun` is used find the path of the Metal compiler, which is varible
|
||||
# and not on $PATH
|
||||
# see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
|
||||
__noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
|
||||
__noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
|
||||
|
||||
nativeBuildInputs =
|
||||
[
|
||||
@@ -181,10 +181,10 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
autoAddDriverRunpath
|
||||
]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
|
||||
++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
|
||||
|
||||
buildInputs =
|
||||
optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
|
||||
optionals effectiveStdenv.isDarwin darwinBuildInputs
|
||||
++ optionals useCuda cudaBuildInputs
|
||||
++ optionals useMpi [ mpi ]
|
||||
++ optionals useRocm rocmBuildInputs
|
||||
@@ -245,7 +245,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
|
||||
# Configurations that are known to result in build failures. Can be
|
||||
# overridden by importing Nixpkgs with `allowBroken = true`.
|
||||
broken = (useMetalKit && !effectiveStdenv.hostPlatform.isDarwin);
|
||||
broken = (useMetalKit && !effectiveStdenv.isDarwin);
|
||||
|
||||
description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
|
||||
homepage = "https://github.com/ggml-org/llama.cpp/";
|
||||
|
||||
+10
-10
@@ -1,18 +1,18 @@
|
||||
ARG OPENVINO_VERSION_MAJOR=2026.4.1
|
||||
ARG OPENVINO_VERSION_FULL=2026.4.1.22982.07f9c262b05
|
||||
ARG OPENVINO_VERSION_MAJOR=2026.3
|
||||
ARG OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
|
||||
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
|
||||
ARG IGC_VERSION=v2.41.5
|
||||
ARG IGC_VERSION_FULL=2_2.41.5+22716
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.35.39758.10
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.35.39758.10-0
|
||||
ARG IGC_VERSION=v2.38.2
|
||||
ARG IGC_VERSION_FULL=2_2.38.2+22051
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.27.39122.11
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.27.39122.11-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
|
||||
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
|
||||
ARG NPU_DRIVER_VERSION=v1.38.0
|
||||
ARG NPU_DRIVER_FULL=v1.38.0.20260910-34487311128
|
||||
ARG LIBZE1_VERSION=1.32.0-1~24.04~ppa1
|
||||
ARG NPU_DRIVER_VERSION=v1.35.0
|
||||
ARG NPU_DRIVER_FULL=v1.35.0.20260722-29947505341
|
||||
ARG LIBZE1_VERSION=1.28.2-1~24.04~ppa1
|
||||
|
||||
# Optional proxy build arguments
|
||||
ARG http_proxy=
|
||||
@@ -173,7 +173,7 @@ RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \
|
||||
fi; \
|
||||
DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \
|
||||
if [ ! -f "$DEB" ]; then \
|
||||
wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260830T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
|
||||
wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260606T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
|
||||
fi; \
|
||||
mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \
|
||||
apt-get update; \
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"Exclude": ["^\\.gitmodules$", "stb_image\\.h", "examples/test-cmake/build/", "examples/test-cmake/build-subdir/"],
|
||||
"Exclude": ["^\\.gitmodules$", "stb_image\\.h"],
|
||||
"Disable": {
|
||||
"IndentSize": true
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
blank_issues_enabled: false
|
||||
blank_issues_enabled: true
|
||||
contact_links:
|
||||
- name: Got an idea?
|
||||
url: https://github.com/ggml-org/llama.cpp/discussions/categories/ideas
|
||||
|
||||
@@ -14,7 +14,7 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
@@ -100,36 +100,36 @@ runs:
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.4 for x64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }}
|
||||
- name: Install Cuda Toolkit 13.3
|
||||
if: ${{ inputs.cuda_version == '13.3' }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3"
|
||||
choco install unzip -y
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
- name: Install Cuda Toolkit 13.4 for ARM64
|
||||
if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }}
|
||||
@@ -137,19 +137,19 @@ runs:
|
||||
run: |
|
||||
mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
choco install unzip -y
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip"
|
||||
curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cccl-windows-x86_64-13.3.4.1.2-archive.zip"
|
||||
curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_crt-windows-x86_64-13.4.46-archive.zip"
|
||||
curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_nvcc-windows-x86_64-13.4.46-archive.zip"
|
||||
curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/libnvvm-windows-x86_64-13.4.46-archive.zip"
|
||||
curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_cudart-windows-arm64-13.4.46-archive.zip"
|
||||
curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/libcublas-windows-arm64-13.7.0.10-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.1.2-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.10-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
|
||||
echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
|
||||
@@ -24,7 +24,7 @@ runs:
|
||||
|
||||
write-host "Installing ROCm wheels for multi-arch support"
|
||||
# Install ROCm wheels for multi-arch support (this may take several minutes)
|
||||
python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ inputs.version }}"
|
||||
python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ inputs.version }}"
|
||||
|
||||
# Pre-expand the devel tree so it is included in the cache
|
||||
write-host "Initializing ROCm devel tree"
|
||||
|
||||
@@ -4,10 +4,6 @@ on:
|
||||
issues:
|
||||
types: [opened]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
find-related:
|
||||
if: github.event.action == 'opened'
|
||||
|
||||
@@ -15,10 +15,6 @@ on:
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
@@ -23,27 +23,19 @@ on:
|
||||
- 'scripts/snapdragon/**'
|
||||
- 'CMakePresets.json'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
android-ndk-snapdragon:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
@@ -67,17 +59,13 @@ jobs:
|
||||
path: pkg-snapdragon/llama.cpp
|
||||
|
||||
linux-iot-snapdragon:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
@@ -110,10 +98,6 @@ jobs:
|
||||
matrix:
|
||||
device: [SM8750, SM8850, QCS9075M]
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
@@ -20,10 +20,6 @@ on:
|
||||
- '.github/workflows/build-android.yml'
|
||||
- 'examples/llama.android/**'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -37,7 +33,7 @@ env:
|
||||
|
||||
jobs:
|
||||
default:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -53,7 +49,7 @@ jobs:
|
||||
distribution: zulu
|
||||
|
||||
- name: Setup Android SDK
|
||||
uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
|
||||
uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
|
||||
with:
|
||||
log-accepted-android-sdk-licenses: false
|
||||
|
||||
@@ -63,17 +59,13 @@ jobs:
|
||||
./gradlew build --no-daemon
|
||||
|
||||
ndk:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3'
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
@@ -101,7 +93,7 @@ jobs:
|
||||
path: pkg-adb/llama.cpp
|
||||
|
||||
arm64:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
env:
|
||||
NDK_VERSION: "29.0.14206865"
|
||||
@@ -118,7 +110,7 @@ jobs:
|
||||
# cache on: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
|
||||
#
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# uses: ggml-org/ccache-action@v1.2.21
|
||||
# with:
|
||||
# key: android-ubuntu-arm64
|
||||
# evict-old-files: 1d
|
||||
@@ -131,7 +123,7 @@ jobs:
|
||||
distribution: temurin
|
||||
|
||||
- name: Setup Android SDK
|
||||
uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
|
||||
uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
|
||||
with:
|
||||
log-accepted-android-sdk-licenses: false
|
||||
|
||||
|
||||
@@ -26,10 +26,6 @@ on:
|
||||
'ggml/src/ggml-rpc/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -37,7 +33,6 @@ concurrency:
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
GGML_SCHED_DEBUG_REALLOC: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
@@ -52,19 +47,11 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: apple-arm64
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -79,33 +66,23 @@ jobs:
|
||||
-DGGML_RPC=ON \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
|
||||
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: apple-arm64
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Check for leaks
|
||||
run: |
|
||||
cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1)
|
||||
leaks -atExit -- "${cmd[@]}"
|
||||
# Graphics devices are leaked by Metal in Apple code sometimes, so we ignore those leaks
|
||||
OBJC_DEBUG_MISSING_POOLS=YES "${cmd[@]}" 2>&1 | awk '{ print } index($0, "autoreleased with no pool in place") && !/class [a-zA-Z0-9]+Device autoreleased/ { found = 1 } END { exit found }'
|
||||
leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
# Metal Paravirtual devices are difficult to support -> disable
|
||||
# ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
|
||||
ctest -L main -E "test-llama-archs|test-save-load-state|test-recurrent-state-rollback" --verbose --timeout 900
|
||||
ctest -L main -E "test-llama-archs" --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: apple-arm64
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
macos-latest-x64:
|
||||
runs-on: macos-15-intel
|
||||
@@ -116,19 +93,11 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: apple-x64
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -145,31 +114,25 @@ jobs:
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
|
||||
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: apple-x64
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: apple-x64
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
macos-latest-ios-xcode:
|
||||
runs-on: macos-latest
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
@@ -267,10 +230,6 @@ jobs:
|
||||
runs-on: macos-latest
|
||||
needs: macos-latest-ios-xcode
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
||||
|
||||
@@ -5,10 +5,6 @@ on:
|
||||
schedule:
|
||||
- cron: '0 * * * *'
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -45,8 +41,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -73,8 +69,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -22,10 +22,6 @@ on:
|
||||
'ggml/src/ggml-cann/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
@@ -3,13 +3,9 @@ on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
linux:
|
||||
runs-on: [self-hosted, Linux, CPU]
|
||||
runs-on: [self-hosted, Linux]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
|
||||
@@ -30,10 +30,6 @@ on:
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -41,7 +37,6 @@ concurrency:
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
GGML_SCHED_DEBUG_REALLOC: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
@@ -67,10 +62,11 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
key: cpu-${{ matrix.os }}
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
@@ -93,16 +89,7 @@ jobs:
|
||||
run: |
|
||||
export PIP_BREAK_SYSTEM_PACKAGES="1"
|
||||
python3 -m pip install --upgrade pip setuptools
|
||||
pip3 install ./gguf-py jinja2==3.1.6
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: cpu-${{ matrix.os }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
pip3 install ./gguf-py
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -113,23 +100,11 @@ jobs:
|
||||
-DGGML_RPC=ON
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: cpu-${{ matrix.os }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L 'main|python' --verbose --timeout 900
|
||||
ctest -L main --verbose --timeout 900
|
||||
|
||||
- name: Test llama2c conversion
|
||||
id: llama2c_test
|
||||
@@ -142,15 +117,22 @@ jobs:
|
||||
./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
|
||||
./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
||||
|
||||
# note: real deletion only on push to master (same condition as the ccache save),
|
||||
# dry-run otherwise (the token is read-only on PRs from forks)
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: cpu-${{ matrix.os }}
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
windows:
|
||||
name: windows / ${{ matrix.build }}
|
||||
runs-on: windows-2025
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
OPENBLAS_VERSION: 0.3.23
|
||||
SDE_VERSION: 9.33.0-2024-01-07
|
||||
@@ -174,7 +156,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: cpu-windows-2025-${{ matrix.build }}
|
||||
variant: ccache
|
||||
@@ -231,6 +213,7 @@ jobs:
|
||||
# 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
|
||||
# $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
|
||||
# cd build
|
||||
# $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
|
||||
# & $sde -future -- ctest -L main -C Release --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
|
||||
@@ -15,10 +15,6 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
@@ -24,10 +24,6 @@ on:
|
||||
'ggml/src/ggml-cuda/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -57,22 +53,22 @@ jobs:
|
||||
apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
key: cuda-ubuntu-24.04-cuda
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: cuda-ubuntu-24.04-cuda
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build with CMake
|
||||
# TODO: Drop GGML_CUDA_CCCL_VERSION when this job uses CTK >= 13.5, which bundles CCCL >= 3.5.
|
||||
# TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
|
||||
run: |
|
||||
cmake -S . -B build -G Ninja \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
@@ -81,7 +77,7 @@ jobs:
|
||||
-DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_CUDA=ON \
|
||||
-DGGML_CUDA_CCCL_VERSION=v3.4.3
|
||||
-DGGML_CUDA_CUB_3DOT2=ON
|
||||
cmake --build build
|
||||
|
||||
- name: ccache-buckets-save
|
||||
@@ -112,15 +108,15 @@ jobs:
|
||||
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
key: cuda-ubuntu-22.04-hip
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: cuda-ubuntu-22.04-hip
|
||||
folder: llama.cpp
|
||||
@@ -149,7 +145,7 @@ jobs:
|
||||
|
||||
musa:
|
||||
runs-on: ubuntu-22.04
|
||||
container: registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu22.04-s5000
|
||||
container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -160,18 +156,18 @@ jobs:
|
||||
id: depends
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y build-essential git cmake libssl-dev jq python3-venv musa-mualg-5-2 musa-muthrust-5-2 libmthreads-compute
|
||||
apt-get install -y build-essential git cmake libssl-dev jq
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
key: cuda-ubuntu-22.04-musa
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: cuda-ubuntu-22.04-musa
|
||||
folder: llama.cpp
|
||||
@@ -181,9 +177,8 @@ jobs:
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build -S . \
|
||||
-DGGML_MUSA=ON \
|
||||
-DMUSA_ARCHITECTURES=31
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
-DGGML_MUSA=ON
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
@@ -7,11 +7,6 @@ name: CI (CUDA, windows)
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
# note: this will run in queue with the release workflow
|
||||
concurrency:
|
||||
group: release
|
||||
@@ -36,16 +31,15 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# CTK >= 13.5 bundles CCCL >= 3.5; omit GGML_CUDA_CCCL_VERSION for those versions.
|
||||
- cuda: '12.4'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
- cuda: '13.4'
|
||||
defines: '-DGGML_CUDA_CUB_3DOT2=ON'
|
||||
- cuda: '13.3'
|
||||
arch: x64
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
|
||||
defines: ''
|
||||
- cuda: '13.4'
|
||||
arch: arm64
|
||||
defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -53,7 +47,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
|
||||
|
||||
@@ -158,7 +152,7 @@ jobs:
|
||||
& "${env:HIP_PATH}\lib\llvm\bin\clang.exe" --version
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
# TODO: this build does not match the build in release.yml, so we use a different cache key
|
||||
# ideally, the builds should match, similar to the CUDA build above so that we would be able
|
||||
|
||||
@@ -19,14 +19,9 @@ on:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-ibm.yml',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
'ggml/src/ggml-zdnn/**'
|
||||
'ggml/src/ggml-cpu/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -39,14 +34,9 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-24-s390x:
|
||||
name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
vxe: ["ON", "OFF"] # `-DGGML_VXE=ON/OFF`
|
||||
ubuntu-24-s390x:
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -87,8 +77,7 @@ jobs:
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DGGML_RPC=ON \
|
||||
-DGGML_VXE=${{ matrix.vxe }}
|
||||
-DGGML_RPC=ON
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Test
|
||||
@@ -105,46 +94,6 @@ jobs:
|
||||
wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
|
||||
./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
||||
|
||||
ubuntu-26-zdnn-s390x:
|
||||
name: ubuntu-26-zdnn-s390x
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
container: ubuntu:26.04 # required to get GCC 15.1 and binutils 2.44
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential cmake git ca-certificates \
|
||||
libssl-dev libzdnn-dev
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Toolchain workaround (GCC 15)
|
||||
run: |
|
||||
apt-get install -y gcc-15 g++-15
|
||||
echo "CC=gcc-15" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-15" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build with zDNN Backend
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_VXE=ON \
|
||||
-DGGML_ZDNN=ON \
|
||||
-DGGML_RPC=ON \
|
||||
-DCMAKE_C_FLAGS="-march=arch15" \
|
||||
-DCMAKE_CXX_FLAGS="-march=arch15"
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
ubuntu-24-ppc64le:
|
||||
runs-on: ubuntu-24.04-ppc64le
|
||||
|
||||
|
||||
@@ -8,10 +8,6 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -39,7 +35,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# uses: ggml-org/ccache-action@v1.2.16
|
||||
# with:
|
||||
# key: msys-windows-2025-x64
|
||||
# variant: ccache
|
||||
|
||||
@@ -23,11 +23,6 @@ on:
|
||||
'ggml/src/ggml-opencl/**'
|
||||
]
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -49,7 +44,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: opencl-windows-2025-x64
|
||||
variant: ccache
|
||||
|
||||
@@ -22,10 +22,6 @@ on:
|
||||
'ggml/src/ggml-openvino/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -36,8 +32,6 @@ env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state"
|
||||
|
||||
jobs:
|
||||
ubuntu-24-openvino:
|
||||
@@ -45,8 +39,8 @@ jobs:
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -84,29 +78,26 @@ jobs:
|
||||
|
||||
- name: Test (CPU)
|
||||
id: cmake_test_cpu
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 2000
|
||||
|
||||
- name: Test (GPU)
|
||||
id: cmake_test_gpu
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
||||
ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 3000
|
||||
|
||||
openvino-windows-2022:
|
||||
runs-on: windows-2022
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -114,7 +105,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: openvino-windows-2022
|
||||
variant: ccache
|
||||
@@ -168,13 +159,14 @@ jobs:
|
||||
- name: Test (CPU)
|
||||
id: cmake_test_cpu
|
||||
shell: cmd
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
run: |
|
||||
REM Find extracted OpenVINO folder dynamically
|
||||
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
|
||||
call "%OPENVINO_ROOT%\setupvars.bat"
|
||||
|
||||
cd build
|
||||
ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
|
||||
ctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
|
||||
@@ -22,10 +22,6 @@ on:
|
||||
'ggml/src/ggml-cpu/arch/riscv/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -71,7 +67,7 @@ jobs:
|
||||
|
||||
# note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
|
||||
# with:
|
||||
# key: riscv-ubuntu-native
|
||||
# evict-old-files: 1d
|
||||
@@ -110,3 +106,83 @@ jobs:
|
||||
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
|
||||
./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
|
||||
./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
||||
|
||||
ubuntu-riscv64-native-sanitizer:
|
||||
runs-on: ubuntu-24.04-riscv
|
||||
|
||||
continue-on-error: true
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
sanitizer: [ADDRESS, THREAD, UNDEFINED]
|
||||
build_type: [Debug]
|
||||
|
||||
steps:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# Set gcc-14 and g++-14 as the default compilers
|
||||
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
|
||||
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
|
||||
|
||||
git lfs install
|
||||
|
||||
- name: GCC version check
|
||||
run: |
|
||||
gcc --version
|
||||
g++ --version
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
|
||||
# with:
|
||||
# key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }}
|
||||
# evict-old-files: 1d
|
||||
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
if: ${{ matrix.sanitizer != 'THREAD' }}
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
|
||||
-DGGML_OPENMP=ON \
|
||||
-DLLAMA_BUILD_EXAMPLES=ON \
|
||||
-DLLAMA_BUILD_TOOLS=ON \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
||||
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
||||
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
||||
-DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
|
||||
-DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
|
||||
|
||||
cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
|
||||
|
||||
- name: Build (no OpenMP)
|
||||
id: cmake_build_no_openmp
|
||||
if: ${{ matrix.sanitizer == 'THREAD' }}
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
|
||||
-DGGML_OPENMP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=ON \
|
||||
-DLLAMA_BUILD_TOOLS=ON \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
||||
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
||||
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
||||
-DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
|
||||
-DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
|
||||
|
||||
cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
|
||||
@@ -21,10 +21,6 @@ on:
|
||||
'.github/workflows/build-sanitize.yml'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -59,7 +55,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# - name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# uses: ggml-org/ccache-action@v1.2.21
|
||||
# if: ${{ matrix.sanitizer != 'UNDEFINED' }}
|
||||
# with:
|
||||
# key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
|
||||
@@ -105,6 +101,8 @@ jobs:
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
# skip run in Debug - very slow
|
||||
if: ${{ matrix.sanitizer != 'UNDEFINED' }}
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main -E tokenizer --verbose --timeout 900
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
name: CI (self-hosted)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/build-self-hosted.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.cu',
|
||||
'**/*.cuh',
|
||||
'**/*.swift',
|
||||
'**/*.m',
|
||||
'**/*.metal',
|
||||
'**/*.comp',
|
||||
'**/*.glsl',
|
||||
'**/*.wgsl'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/build-self-hosted.yml',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.cu',
|
||||
'**/*.cuh',
|
||||
'**/*.swift',
|
||||
'**/*.m',
|
||||
'**/*.metal',
|
||||
'**/*.comp',
|
||||
'**/*.glsl',
|
||||
'**/*.wgsl'
|
||||
]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
gpu-cuda:
|
||||
runs-on: [self-hosted, Linux, NVIDIA]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
nvidia-smi
|
||||
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-rocm:
|
||||
runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
# HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
|
||||
# issue on integrated RDNA3.5 (gfx1151) where batched inference returns
|
||||
# incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
|
||||
# restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
|
||||
env:
|
||||
HIP_LAUNCH_BLOCKING: "1"
|
||||
run: |
|
||||
rocminfo
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-nvidia-cm:
|
||||
runs-on: [self-hosted, Linux, NVIDIA]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-nvidia-cm2:
|
||||
runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-webgpu-nvidia:
|
||||
runs-on: [self-hosted, Linux, NVIDIA, X64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
DAWN_VERSION="v20260317.182325"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_WEBGPU=1 \
|
||||
GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
|
||||
GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMX-compatible machine
|
||||
#cpu-amx:
|
||||
# runs-on: [self-hosted, Linux, CPU, AMX]
|
||||
|
||||
# steps:
|
||||
# - name: Clone
|
||||
# id: checkout
|
||||
# uses: actions/checkout@v6
|
||||
|
||||
# - name: Test
|
||||
# id: ggml-ci
|
||||
# run: |
|
||||
# bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMD GPU machine
|
||||
# amd-vulkan:
|
||||
# runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
# steps:
|
||||
# - name: Clone
|
||||
# id: checkout
|
||||
# uses: actions/checkout@v6
|
||||
|
||||
# - name: Test
|
||||
# id: ggml-ci
|
||||
# run: |
|
||||
# vulkaninfo --summary
|
||||
# GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMD GPU machine
|
||||
# amd-rocm:
|
||||
# runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
# steps:
|
||||
# - name: Clone
|
||||
# id: checkout
|
||||
# uses: actions/checkout@v6
|
||||
|
||||
# - name: Test
|
||||
# id: ggml-ci
|
||||
# run: |
|
||||
# amd-smi static
|
||||
# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-metal:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-webgpu-apple:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
DAWN_VERSION="v20260317.182325"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-apple:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-intel-linux:
|
||||
runs-on: [self-hosted, Linux, Intel]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-intel-windows:
|
||||
runs-on: [self-hosted, Windows, X64, Intel]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
|
||||
env:
|
||||
MSYSTEM: UCRT64
|
||||
CHERE_INVOKING: 1
|
||||
PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
# Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
|
||||
# a valid python environment for testing
|
||||
LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
|
||||
|
||||
gpu-openvino-low-perf:
|
||||
runs-on: [self-hosted, Linux, Intel, OpenVINO]
|
||||
|
||||
env:
|
||||
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.3"
|
||||
OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup OpenVINO Toolkit
|
||||
uses: ./.github/actions/linux-setup-openvino
|
||||
with:
|
||||
path: ./openvino_toolkit
|
||||
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
||||
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
||||
|
||||
- name: Install OpenVINO dependencies
|
||||
run: |
|
||||
cd ./openvino_toolkit
|
||||
chmod +x ./install_dependencies/install_openvino_dependencies.sh
|
||||
echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
source ./openvino_toolkit/setupvars.sh
|
||||
GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
cpu-x64-high-perf:
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
cpu-arm64-high-perf-graviton4:
|
||||
runs-on: ah-ubuntu_22_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
sudo apt-get update
|
||||
sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
|
||||
apt-get install -y \
|
||||
build-essential \
|
||||
python3-venv \
|
||||
gpg \
|
||||
wget \
|
||||
time \
|
||||
git-lfs
|
||||
|
||||
git lfs install
|
||||
|
||||
# install the latest cmake
|
||||
sudo install -d /usr/share/keyrings
|
||||
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
|
||||
| gpg --dearmor \
|
||||
| sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
|
||||
| sudo tee /etc/apt/sources.list.d/kitware.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y cmake
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
cpu-arm64-graviton4-kleidiai:
|
||||
runs-on: ah-ubuntu_22_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
sudo apt-get update
|
||||
sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
|
||||
apt-get install -y \
|
||||
build-essential \
|
||||
python3-venv \
|
||||
gpg \
|
||||
wget \
|
||||
time \
|
||||
git-lfs
|
||||
|
||||
git lfs install
|
||||
|
||||
# install the latest cmake
|
||||
sudo install -d /usr/share/keyrings
|
||||
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
|
||||
| gpg --dearmor \
|
||||
| sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
|
||||
| sudo tee /etc/apt/sources.list.d/kitware.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y cmake
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_KLEIDIAI=1 \
|
||||
GG_BUILD_EXTRA_TESTS_0=1 \
|
||||
bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
@@ -22,10 +22,6 @@ on:
|
||||
'ggml/src/ggml-sycl/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -52,8 +48,8 @@ jobs:
|
||||
|
||||
env:
|
||||
ONEAPI_ROOT: /opt/intel/oneapi/
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
LEVEL_ZERO_VERSION: "1.33.1"
|
||||
ONEAPI_INSTALLER_VERSION: "2025.3.3"
|
||||
LEVEL_ZERO_VERSION: "1.28.2"
|
||||
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
||||
|
||||
continue-on-error: true
|
||||
@@ -67,32 +63,23 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
cd /tmp
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
|
||||
sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
|
||||
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
|
||||
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
||||
|
||||
- name: Install Level Zero SDK
|
||||
shell: bash
|
||||
run: |
|
||||
cd /tmp
|
||||
# v1.33.x renamed the Debian packages to libze1 / libze-dev
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb
|
||||
sudo apt-get install -y ./libze1.deb ./libze-dev.deb
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
|
||||
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
|
||||
sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: sycl-ubuntu-24-${{ matrix.build }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -106,43 +93,32 @@ jobs:
|
||||
-DCMAKE_CXX_COMPILER=icpx \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_SYCL_F16=${{ matrix.fp16 }} \
|
||||
-DGGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON \
|
||||
-DGGML_SYCL_DNN=ON \
|
||||
-DCMAKE_CXX_FLAGS="-fsycl-unnamed-lambda" \
|
||||
-DCMAKE_EXE_LINKER_FLAGS="-fsycl-unnamed-lambda"
|
||||
-DGGML_SYCL_F16=${{ matrix.fp16 }}
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: sycl-ubuntu-24-${{ matrix.build }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
windows-latest-sycl:
|
||||
runs-on: windows-2022
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
|
||||
WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
|
||||
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
|
||||
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
|
||||
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
||||
ONEAPI_INSTALLER_VERSION: "2026.1"
|
||||
ONEAPI_INSTALLER_VERSION: "2025.3.3"
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
@@ -161,7 +137,7 @@ jobs:
|
||||
"LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: sycl-windows-latest
|
||||
variant: ccache
|
||||
|
||||
@@ -22,10 +22,6 @@ on:
|
||||
'ggml/src/ggml-virtgpu/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
@@ -24,10 +24,6 @@ on:
|
||||
'ggml/src/ggml-vulkan/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -35,7 +31,6 @@ concurrency:
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
GGML_SCHED_DEBUG_REALLOC: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
@@ -58,20 +53,12 @@ jobs:
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
variant: ccache
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: vulkan-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
variant: ccache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Configure
|
||||
id: cmake_configure
|
||||
@@ -86,17 +73,15 @@ jobs:
|
||||
run: |
|
||||
time cmake --build build -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: vulkan-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
ubuntu-llvmpipe:
|
||||
runs-on: ubuntu-24.04
|
||||
@@ -127,19 +112,11 @@ jobs:
|
||||
strip: 1
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: vulkan-ubuntu-24.04-llvmpipe
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -150,18 +127,6 @@ jobs:
|
||||
-DGGML_VULKAN=ON
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: vulkan-ubuntu-24.04-llvmpipe
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
@@ -169,23 +134,23 @@ jobs:
|
||||
export GGML_VK_VISIBLE_DEVICES=0
|
||||
export GGML_VK_DISABLE_F16=1
|
||||
export GGML_VK_DISABLE_COOPMAT=1
|
||||
ctest -L main --verbose --timeout 900
|
||||
# This is using llvmpipe and runs slower than other backends
|
||||
# test-backend-ops is too slow on llvmpipe, skip it
|
||||
ctest -L main -E test-backend-ops --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: vulkan-ubuntu-24.04-llvmpipe
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
windows:
|
||||
name: windows / ${{ matrix.arch }}
|
||||
runs-on: windows-2025
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- arch: 'x64'
|
||||
- arch: 'arm64'
|
||||
|
||||
env:
|
||||
VULKAN_VERSION: 1.4.357.0
|
||||
|
||||
@@ -195,9 +160,9 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: cpu-windows-2025-${{ matrix.arch }}-vulkan
|
||||
key: cpu-windows-2025-x64-vulkan
|
||||
variant: ccache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
@@ -206,7 +171,7 @@ jobs:
|
||||
id: get_vulkan
|
||||
run: |
|
||||
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
|
||||
& "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install ${{ matrix.arch == 'arm64' && 'com.lunarg.vulkan.arm64' || '' }}
|
||||
& "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
|
||||
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
|
||||
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
|
||||
|
||||
@@ -219,19 +184,18 @@ jobs:
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -S . -B build -G "Ninja Multi-Config" `
|
||||
-D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake `
|
||||
-D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `
|
||||
-DCMAKE_BUILD_TYPE=Release `
|
||||
-DGGML_NATIVE=OFF `
|
||||
-DLLAMA_BUILD_SERVER=ON `
|
||||
-DGGML_RPC=ON `
|
||||
-DGGML_BACKEND_DL=ON `
|
||||
-DGGML_CPU_ALL_VARIANTS=${{ matrix.arch == 'x64' && 'ON' || 'OFF' }} `
|
||||
-DGGML_CPU_ALL_VARIANTS=ON `
|
||||
-DGGML_VULKAN=ON `
|
||||
-DLLAMA_BUILD_BORINGSSL=ON
|
||||
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
|
||||
|
||||
- name: Test
|
||||
if: ${{ matrix.arch == 'x64' }}
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
@@ -242,7 +206,7 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: cpu-windows-2025-${{ matrix.arch }}-vulkan
|
||||
key: cpu-windows-2025-x64-vulkan
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
@@ -33,10 +33,6 @@ on:
|
||||
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -58,10 +54,11 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
key: webgpu-ubuntu-24.04-arm-wasm
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install Emscripten
|
||||
run: |
|
||||
@@ -72,22 +69,13 @@ jobs:
|
||||
|
||||
- name: Fetch emdawnwebgpu
|
||||
run: |
|
||||
DAWN_TAG="v20260908.214631"
|
||||
DAWN_TAG="v20260317.182325"
|
||||
EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip"
|
||||
echo "Downloading ${EMDAWN_PKG}"
|
||||
curl -L -o emdawn.zip \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"
|
||||
unzip emdawn.zip
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04-arm-wasm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build WASM WebGPU
|
||||
run: |
|
||||
source emsdk/emsdk_env.sh
|
||||
@@ -101,14 +89,12 @@ jobs:
|
||||
|
||||
time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04-arm-wasm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
@@ -25,10 +25,6 @@ on:
|
||||
'ggml/src/ggml-webgpu/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -73,33 +69,25 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
key: webgpu-macos-latest
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
DAWN_VERSION="v20260908.214631"
|
||||
DAWN_VERSION="v20260317.182325"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
|
||||
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-macos-latest
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
@@ -107,24 +95,22 @@ jobs:
|
||||
cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
|
||||
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-macos-latest
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: webgpu-macos-latest
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
ubuntu:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
@@ -134,10 +120,11 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
key: webgpu-ubuntu-24.04
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -151,25 +138,16 @@ jobs:
|
||||
id: dawn-depends
|
||||
run: |
|
||||
sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
|
||||
DAWN_VERSION="v20260908.214631"
|
||||
DAWN_VERSION="v20260317.182325"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
|
||||
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
@@ -178,20 +156,20 @@ jobs:
|
||||
-DGGML_WEBGPU=ON
|
||||
time cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Test
|
||||
id: cmake_test
|
||||
run: |
|
||||
cd build
|
||||
ctest -L main --verbose --timeout 900
|
||||
# This is using llvmpipe and runs slower than other backends
|
||||
# test-backend-ops is too slow on llvmpipe, skip it
|
||||
ctest -L main -E test-backend-ops --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: webgpu-ubuntu-24.04
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
@@ -17,13 +17,9 @@ on:
|
||||
'scripts/sync_vendor.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check-vendor:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: [self-hosted, fast]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
name: CI (self-hosted CPU backend)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-cpu.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-cpu.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
cpu-x64-high-perf:
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
cpu-arm64-high-perf-graviton4:
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
sudo apt-get update
|
||||
sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
|
||||
apt-get install -y \
|
||||
build-essential \
|
||||
python3-venv \
|
||||
gpg \
|
||||
wget \
|
||||
time \
|
||||
git-lfs
|
||||
|
||||
git lfs install
|
||||
|
||||
# install the latest cmake
|
||||
sudo install -d /usr/share/keyrings
|
||||
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
|
||||
| gpg --dearmor \
|
||||
| sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
|
||||
| sudo tee /etc/apt/sources.list.d/kitware.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y cmake
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) \
|
||||
GG_BUILD_HIGH_PERF=1 \
|
||||
GG_BUILD_NO_BF16=1 \
|
||||
GG_BUILD_EXTRA_TESTS_0=1 \
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMX-compatible machine
|
||||
#cpu-amx:
|
||||
# runs-on: [self-hosted, Linux, CPU, AMX]
|
||||
|
||||
# steps:
|
||||
# - name: Clone
|
||||
# id: checkout
|
||||
# uses: actions/checkout@v6
|
||||
|
||||
# - name: Test
|
||||
# id: ggml-ci
|
||||
# run: |
|
||||
# bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -1,128 +0,0 @@
|
||||
name: CI (self-hosted CUDA backend)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-cuda.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.cu',
|
||||
'**/*.cuh'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-cuda.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
'ggml/src/ggml-cuda/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
gpu-cuda:
|
||||
runs-on: "hf-jobs-t4-medium:cuda13"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-gpu-cuda
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
nvidia-smi
|
||||
GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-gpu-cuda
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
gpu-rocm:
|
||||
runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
# HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
|
||||
# issue on integrated RDNA3.5 (gfx1151) where batched inference returns
|
||||
# incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
|
||||
# restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
|
||||
env:
|
||||
HIP_LAUNCH_BLOCKING: "1"
|
||||
run: |
|
||||
rocminfo
|
||||
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMD GPU machine
|
||||
# amd-rocm:
|
||||
# runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
# steps:
|
||||
# - name: Clone
|
||||
# id: checkout
|
||||
# uses: actions/checkout@v6
|
||||
|
||||
# - name: Test
|
||||
# id: ggml-ci
|
||||
# run: |
|
||||
# amd-smi static
|
||||
# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -1,89 +0,0 @@
|
||||
name: CI (self-hosted KleidiAI backend)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-kleidiai.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-kleidiai.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
cpu-arm64-graviton4-kleidiai:
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
sudo apt-get update
|
||||
sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
|
||||
apt-get install -y \
|
||||
build-essential \
|
||||
python3-venv \
|
||||
gpg \
|
||||
wget \
|
||||
time \
|
||||
git-lfs
|
||||
|
||||
git lfs install
|
||||
|
||||
# install the latest cmake
|
||||
sudo install -d /usr/share/keyrings
|
||||
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
|
||||
| gpg --dearmor \
|
||||
| sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
||||
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
|
||||
| sudo tee /etc/apt/sources.list.d/kitware.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y cmake
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) \
|
||||
GG_BUILD_KLEIDIAI=1 \
|
||||
GG_BUILD_EXTRA_TESTS_0=1 \
|
||||
GG_BUILD_HIGH_PERF=1 \
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -1,63 +0,0 @@
|
||||
name: CI (self-hosted Metal backend)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-metal.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.swift',
|
||||
'**/*.m',
|
||||
'**/*.metal'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-metal.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
'ggml/src/ggml-metal/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
gpu-metal:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -1,79 +0,0 @@
|
||||
name: CI (self-hosted OpenVINO backend)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-openvino.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-openvino.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
'ggml/src/ggml-openvino/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
gpu-openvino-low-perf:
|
||||
runs-on: [self-hosted, Linux, Intel, OpenVINO]
|
||||
|
||||
env:
|
||||
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
||||
OPENVINO_VERSION_MAJOR: "2026.4.1"
|
||||
OPENVINO_VERSION_FULL: "2026.4.1.22982.07f9c262b05"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup OpenVINO Toolkit
|
||||
uses: ./.github/actions/linux-setup-openvino
|
||||
with:
|
||||
path: ./openvino_toolkit
|
||||
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
||||
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
||||
|
||||
- name: Install OpenVINO dependencies
|
||||
run: |
|
||||
cd ./openvino_toolkit
|
||||
chmod +x ./install_dependencies/install_openvino_dependencies.sh
|
||||
echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
source ./openvino_toolkit/setupvars.sh
|
||||
GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -1,205 +0,0 @@
|
||||
name: CI (self-hosted Vulkan backend)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-vulkan.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.comp',
|
||||
'**/*.glsl'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-vulkan.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
'ggml/src/ggml-vulkan/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
gpu-vulkan-nvidia-cm:
|
||||
# runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
runs-on: [self-hosted, Linux, NVIDIA]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# - name: Install dependencies
|
||||
# run: |
|
||||
# sudo apt update
|
||||
# sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
|
||||
# - name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# with:
|
||||
# restore: false
|
||||
# save: false
|
||||
|
||||
# - name: ccache-buckets-restore
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm
|
||||
# folder: llama.cpp
|
||||
# hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# - name: ccache-buckets-save
|
||||
# if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# env:
|
||||
# HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm
|
||||
# folder: llama.cpp
|
||||
# evict-old-files: 1d
|
||||
# hf_bucket: ggml-org/cache
|
||||
# save: true
|
||||
|
||||
gpu-vulkan-nvidia-cm2:
|
||||
# runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
# - name: Install dependencies
|
||||
# run: |
|
||||
# sudo apt update
|
||||
# sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
|
||||
# - name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
# with:
|
||||
# restore: false
|
||||
# save: false
|
||||
|
||||
# - name: ccache-buckets-restore
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm2
|
||||
# folder: llama.cpp
|
||||
# hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
# - name: ccache-buckets-save
|
||||
# if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
# uses: ./.github/actions/ccache-buckets
|
||||
# env:
|
||||
# HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
# with:
|
||||
# key: self-hosted-vulkan-nvidia-cm2
|
||||
# folder: llama.cpp
|
||||
# evict-old-files: 1d
|
||||
# hf_bucket: ggml-org/cache
|
||||
# save: true
|
||||
|
||||
gpu-vulkan-apple:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-intel-linux:
|
||||
runs-on: [self-hosted, Linux, Intel]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
gpu-vulkan-intel-windows:
|
||||
runs-on: [self-hosted, Windows, X64, Intel]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
|
||||
env:
|
||||
MSYSTEM: UCRT64
|
||||
CHERE_INVOKING: 1
|
||||
PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
# Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
|
||||
# a valid python environment for testing
|
||||
LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
|
||||
|
||||
# TODO: provision AMD GPU machine
|
||||
# amd-vulkan:
|
||||
# runs-on: [self-hosted, Linux, AMD]
|
||||
|
||||
# steps:
|
||||
# - name: Clone
|
||||
# id: checkout
|
||||
# uses: actions/checkout@v6
|
||||
|
||||
# - name: Test
|
||||
# id: ggml-ci
|
||||
# run: |
|
||||
# vulkaninfo --summary
|
||||
# GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -1,134 +0,0 @@
|
||||
name: CI (self-hosted WebGPU backend)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-webgpu.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.wgsl'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ci-self-hosted-webgpu.yml',
|
||||
'ci/run.sh',
|
||||
'**/CMakeLists.txt',
|
||||
'**/.cmake',
|
||||
'ggml/src/*',
|
||||
'ggml/src/ggml-cpu/**',
|
||||
'ggml/src/ggml-webgpu/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
gpu-webgpu-nvidia:
|
||||
runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-webgpu-nvidia
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
DAWN_VERSION="v20260908.214631"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_WEBGPU=1 \
|
||||
GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
|
||||
GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-webgpu-nvidia
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
gpu-webgpu-apple:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
DAWN_VERSION="v20260908.214631"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
|
||||
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
||||
@@ -3,9 +3,10 @@ on:
|
||||
schedule:
|
||||
- cron: "42 0 * * *"
|
||||
|
||||
cache-mode: none
|
||||
# Fine-grant permission
|
||||
# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
close-issues:
|
||||
|
||||
@@ -9,17 +9,13 @@ on:
|
||||
branches:
|
||||
- master
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
model-naming:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: [self-hosted, fast]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Check model naming conventions
|
||||
|
||||
@@ -11,12 +11,10 @@ on:
|
||||
paths:
|
||||
- .github/workflows/copilot-setup-steps.yml
|
||||
|
||||
cache-mode: none
|
||||
|
||||
jobs:
|
||||
# The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
|
||||
copilot-setup-steps:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Set the permissions to the lowest permissions possible needed for your steps.
|
||||
# Copilot will be given its own token for its operations.
|
||||
@@ -31,10 +29,10 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
key: copilot-setup-steps
|
||||
evict-old-files: 1d
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
|
||||
@@ -20,14 +20,15 @@ on:
|
||||
# Rebuild daily rather than on every push because it is expensive
|
||||
- cron: '12 4 * * *'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# Fine-grant permission
|
||||
# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
|
||||
permissions:
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
create_tag:
|
||||
name: Create and push git tag
|
||||
@@ -61,9 +62,6 @@ jobs:
|
||||
build_ui:
|
||||
name: Build UI
|
||||
needs: create_tag
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
uses: ./.github/workflows/ui-build.yml
|
||||
with:
|
||||
ui_version: ${{ needs.create_tag.outputs.source_tag }}
|
||||
@@ -92,8 +90,8 @@ jobs:
|
||||
{ "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
|
||||
{ "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
|
||||
{ "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },
|
||||
@@ -148,11 +146,6 @@ jobs:
|
||||
needs: [prepare_matrices, create_tag, build_ui]
|
||||
|
||||
runs-on: ${{ matrix.config.runs_on }}
|
||||
# cache-mode: write # for QEMU
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
packages: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -172,11 +165,11 @@ jobs:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
|
||||
# - name: Set up QEMU
|
||||
# if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
# uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
|
||||
# with:
|
||||
# image: tonistiigi/binfmt:qemu-v10.2.1
|
||||
- name: Set up QEMU
|
||||
if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
|
||||
with:
|
||||
image: tonistiigi/binfmt:qemu-v10.2.1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
|
||||
|
||||
@@ -9,17 +9,13 @@ on:
|
||||
branches:
|
||||
- master
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
editorconfig:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: [self-hosted, fast]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: editorconfig-checker/action-editorconfig-checker@840e866d93b8e032123c23bac69dece044d4d84c # v2.2.0
|
||||
|
||||
@@ -17,14 +17,11 @@ on:
|
||||
tags:
|
||||
- 'gguf-v*' # Push events to every version tag
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
@@ -25,10 +25,6 @@ on:
|
||||
'scripts/hip/gcn-cdna-vgpr-check.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
@@ -53,22 +49,14 @@ jobs:
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3 python3-venv python3-pip jq
|
||||
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: hip-quality-check-ubuntu-22.04
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build with Werror
|
||||
id: cmake_build
|
||||
@@ -97,14 +85,12 @@ jobs:
|
||||
make -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$'
|
||||
python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: hip-quality-check-ubuntu-22.04
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
@@ -2,10 +2,6 @@ name: "Pull Request Labeler"
|
||||
on:
|
||||
- pull_request_target
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
labeler:
|
||||
permissions:
|
||||
|
||||
@@ -13,33 +13,16 @@ on:
|
||||
required: true
|
||||
type: boolean
|
||||
default: true
|
||||
skip_apiabi_check:
|
||||
description: 'Skip API/ABI compatibility check'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
require_docker:
|
||||
description: 'Require the Docker workflow to have completed successfully'
|
||||
required: true
|
||||
type: boolean
|
||||
default: true
|
||||
apiabi_compare_tag:
|
||||
description: 'Tag to compare against for API/ABI check (default: latest release)'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
make-release:
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -49,19 +32,12 @@ jobs:
|
||||
ref: ${{ inputs.commit != '' && inputs.commit || github.ref_name }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install API/ABI check tools
|
||||
if: ${{ github.event.inputs.skip_apiabi_check != 'true' }}
|
||||
run: sudo apt-get install -y abi-compliance-checker abigail-tools
|
||||
|
||||
- name: Run release checks
|
||||
id: checks
|
||||
run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}
|
||||
env:
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
RELEASE_BRANCH: ${{ github.ref_name }}
|
||||
SKIP_APIABI_CHECK: ${{ github.event.inputs.skip_apiabi_check }}
|
||||
APIABI_COMPARE_TAG: ${{ github.event.inputs.apiabi_compare_tag }}
|
||||
REQUIRE_DOCKER: ${{ github.event.inputs.require_docker }}
|
||||
|
||||
- name: Create release tag
|
||||
if: ${{ github.event.inputs.dry_run == 'false' }}
|
||||
@@ -137,38 +113,6 @@ jobs:
|
||||
data: await fs.readFileSync('./nightly-tag.txt')
|
||||
});
|
||||
|
||||
- name: Re-tag container images with release version
|
||||
if: ${{ github.event.inputs.dry_run == 'false' && github.event.inputs.require_docker != 'false' && steps.desc.outputs.nightly_tag != '' }}
|
||||
env:
|
||||
GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
|
||||
run: |
|
||||
VERSION="${{ steps.checks.outputs.version }}"
|
||||
NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}"
|
||||
REPO_OWNER="${GITHUB_REPOSITORY_OWNER,,}"
|
||||
IMAGE_REPO="ghcr.io/${REPO_OWNER}/${{ github.event.repository.name }}"
|
||||
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
VARIANTS=("" "-cuda" "-cuda13" "-vulkan" "-rocm" "-intel" "-musa" "-openvino")
|
||||
TYPES=("full" "light" "server")
|
||||
# the release is already created at this point, so keep going on a
|
||||
# missing image and report all of them at the end
|
||||
MISSING=()
|
||||
for type in "${TYPES[@]}"; do
|
||||
for variant in "${VARIANTS[@]}"; do
|
||||
src="${IMAGE_REPO}:${type}${variant}-${NIGHTLY_TAG}"
|
||||
dst="${IMAGE_REPO}:${type}${variant}-${VERSION}"
|
||||
echo "Tagging ${src} -> ${dst}"
|
||||
if ! docker buildx imagetools create --tag "${dst}" "${src}"; then
|
||||
MISSING+=("${type}${variant}")
|
||||
fi
|
||||
done
|
||||
done
|
||||
if [[ ${#MISSING[@]} -gt 0 ]]; then
|
||||
echo "::error::failed to re-tag container images for ${NIGHTLY_TAG}:${MISSING[*]}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Dry run summary
|
||||
if: ${{ github.event.inputs.dry_run == 'true' }}
|
||||
run: |
|
||||
|
||||
@@ -1,449 +0,0 @@
|
||||
name: Models Backend Check
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/models-check.yml',
|
||||
'ggml/**',
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/llama-graph.cpp',
|
||||
'src/llama-model*',
|
||||
'src/models/**'
|
||||
]
|
||||
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/models-check.yml',
|
||||
'ggml/**',
|
||||
'tests/fusion/**',
|
||||
'tests/test-fusion.cpp',
|
||||
'tests/test-llama-archs.cpp',
|
||||
'src/llama-graph.cpp',
|
||||
'src/llama-model*',
|
||||
'src/models/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
cuda:
|
||||
runs-on: "hf-jobs-t4-medium:cuda13"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y cmake time python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: models-check-cuda
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_SCHED_NO_REALLOC=ON \
|
||||
-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \
|
||||
-DGGML_CUDA=ON
|
||||
time cmake --build build --config Release --target test-llama-archs -j$(nproc)
|
||||
time cmake --build build --config Release --target test-fusion -j$(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: models-check-cuda
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
# - name: Generate models
|
||||
# id: generate_models
|
||||
# run: |
|
||||
# rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
# ./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
# TODO: add for backends as they adopt the fusion debug API
|
||||
# - name: Test fusion
|
||||
# id: test_fusion
|
||||
# run: |
|
||||
# ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1
|
||||
GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1
|
||||
GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1
|
||||
GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1
|
||||
|
||||
metal:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_SCHED_NO_REALLOC=ON \
|
||||
-DGGML_BLAS=OFF \
|
||||
-DGGML_METAL=ON
|
||||
time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)
|
||||
time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
- name: Generate models
|
||||
id: generate_models
|
||||
run: |
|
||||
rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
- name: Test fusion
|
||||
id: test_fusion
|
||||
run: |
|
||||
./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
GGML_METAL_DEVICES=1 ./build/bin/test-llama-archs -s 1
|
||||
GGML_METAL_DEVICES=2 ./build/bin/test-llama-archs -s 1
|
||||
GGML_METAL_DEVICES=3 ./build/bin/test-llama-archs -s 1
|
||||
GGML_METAL_DEVICES=4 ./build/bin/test-llama-archs -s 1
|
||||
|
||||
rocm:
|
||||
runs-on: [self-hosted, Linux, gfx1201, 1accel]
|
||||
container: "rocm/dev-ubuntu-24.04:7.2.4-complete"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt update
|
||||
apt install -y build-essential jq cmake time python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: models-check-rocm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_SCHED_NO_REALLOC=ON \
|
||||
-DCMAKE_HIP_COMPILER=$(hipconfig -l)/clang \
|
||||
-DGPU_TARGETS=gfx1201 \
|
||||
-DGGML_HIP=ON
|
||||
time cmake --build build --config Release --target test-llama-archs -j$(nproc)
|
||||
time cmake --build build --config Release --target test-fusion -j$(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: models-check-rocm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
# - name: Generate models
|
||||
# id: generate_models
|
||||
# run: |
|
||||
# rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
# ./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
# TODO: add for backends as they adopt the fusion debug API
|
||||
# - name: Test fusion
|
||||
# id: test_fusion
|
||||
# run: |
|
||||
# ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1
|
||||
GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1
|
||||
GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1
|
||||
GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1
|
||||
|
||||
vulkan-nvidia:
|
||||
runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: models-check-vulkan-nvidia
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_SCHED_NO_REALLOC=ON \
|
||||
-DGGML_VULKAN=ON
|
||||
time cmake --build build --config Release --target test-llama-archs -j$(nproc)
|
||||
time cmake --build build --config Release --target test-fusion -j$(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: models-check-vulkan-nvidia
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
# - name: Generate models
|
||||
# id: generate_models
|
||||
# run: |
|
||||
# rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
# ./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
# TODO: add for backends as they adopt the fusion debug API
|
||||
# - name: Test fusion
|
||||
# id: test_fusion
|
||||
# run: |
|
||||
# ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
./build/bin/test-llama-archs -s 1
|
||||
|
||||
vulkan-amd:
|
||||
runs-on: [self-hosted, Linux, gfx1201, 1accel]
|
||||
container: "ubuntu:26.04"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt update
|
||||
apt install -y build-essential jq cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: models-check-vulkan-amd
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_SCHED_NO_REALLOC=ON \
|
||||
-DGGML_VULKAN=ON
|
||||
time cmake --build build --config Release --target test-llama-archs -j$(nproc)
|
||||
time cmake --build build --config Release --target test-fusion -j$(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: models-check-vulkan-amd
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
# - name: Generate models
|
||||
# id: generate_models
|
||||
# run: |
|
||||
# rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
# ./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
# TODO: add for backends as they adopt the fusion debug API
|
||||
# - name: Test fusion
|
||||
# id: test_fusion
|
||||
# run: |
|
||||
# ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
./build/bin/test-llama-archs -s 1
|
||||
|
||||
webgpu-nvidia:
|
||||
runs-on: "hf-jobs-t4-small:ubuntu26_04"
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: models-check-webgpu-nvidia
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Dawn Dependency
|
||||
id: dawn-depends
|
||||
run: |
|
||||
DAWN_VERSION="v20260908.214631"
|
||||
DAWN_OWNER="google"
|
||||
DAWN_REPO="dawn"
|
||||
DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
|
||||
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
curl -L -o artifact.tar.gz \
|
||||
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
||||
mkdir dawn
|
||||
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DGGML_SCHED_NO_REALLOC=ON \
|
||||
-DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/dawn" \
|
||||
-DDawn_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
|
||||
-DGGML_WEBGPU=ON
|
||||
time cmake --build build --config Release --target test-llama-archs -j$(nproc)
|
||||
time cmake --build build --config Release --target test-fusion -j$(nproc)
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: models-check-webgpu-nvidia
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
# - name: Generate models
|
||||
# id: generate_models
|
||||
# run: |
|
||||
# rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
# ./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
# TODO: add for backends as they adopt the fusion debug API
|
||||
# - name: Test fusion
|
||||
# id: test_fusion
|
||||
# run: |
|
||||
# ./build/bin/test-fusion --models build-ci-models --device WebGPU --check tests/fusion/WebGPU.csv
|
||||
|
||||
- name: Test archs
|
||||
id: test_archs
|
||||
run: |
|
||||
./build/bin/test-llama-archs -s 1
|
||||
@@ -4,7 +4,6 @@ on:
|
||||
pull_request_target:
|
||||
types: [labeled]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
@@ -10,13 +10,9 @@ on:
|
||||
- 'conversion/base.py'
|
||||
- 'convert_hf_to_gguf_update.py'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pre-tokenizer-hashes:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: [self-hosted, fast]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -14,17 +14,13 @@ on:
|
||||
- 'convert*.py'
|
||||
- '**/requirements*.txt'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
python-check-requirements:
|
||||
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
||||
runs-on: [self-hosted, CPU, fast]
|
||||
name: check-requirements
|
||||
steps:
|
||||
- name: Check out source repository
|
||||
|
||||
@@ -15,17 +15,13 @@ on:
|
||||
'**/*.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
flake8-lint:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: [self-hosted, fast]
|
||||
name: Lint
|
||||
steps:
|
||||
- name: Check out source repository
|
||||
|
||||
@@ -16,17 +16,13 @@ on:
|
||||
- '**/requirements*.txt'
|
||||
# - 'pyrightconfig.json'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
python-type-check:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: [self-hosted, fast]
|
||||
name: python type-check
|
||||
steps:
|
||||
- name: Check out source repository
|
||||
@@ -35,7 +31,7 @@ jobs:
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.84
|
||||
pip-install: -r requirements/requirements-all.txt ty==0.0.35
|
||||
# - name: Type-check with Pyright
|
||||
# uses: jakebailey/pyright-action@v2
|
||||
# with:
|
||||
@@ -44,4 +40,4 @@ jobs:
|
||||
# warnings: true
|
||||
- name: Type-check with ty
|
||||
run: |
|
||||
ty check --exit-zero-on-warning --output-format=github
|
||||
ty check --output-format=github
|
||||
|
||||
@@ -1,245 +0,0 @@
|
||||
name: Publish Release
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- Release
|
||||
types:
|
||||
- completed
|
||||
branches:
|
||||
- master
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BRANCH_NAME: master
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
|
||||
# Fine-grained permission
|
||||
# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write # for creating release
|
||||
id-token: write
|
||||
attestations: write
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
outputs:
|
||||
should_release: ${{ steps.check.outputs.should_release }}
|
||||
tag_name: ${{ steps.tag.outputs.name }}
|
||||
|
||||
steps:
|
||||
- id: check
|
||||
env:
|
||||
COMMIT_MESSAGE: ${{ github.event.workflow_run.head_commit.message }}
|
||||
run: |
|
||||
if echo "$COMMIT_MESSAGE" | grep -q '\[no release\]'; then
|
||||
echo "should_release=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "should_release=true" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Clone
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
fetch-depth: 0
|
||||
ssh-key: ${{ secrets.DEPLOY_KEY_RELEASE }}
|
||||
|
||||
- name: Determine tag name
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: tag
|
||||
uses: ./.github/actions/get-tag-name
|
||||
|
||||
- name: Download artifacts
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: download-artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: ./artifact
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
github-token: ${{ github.token }}
|
||||
merge-multiple: true
|
||||
skip-decompress: true
|
||||
|
||||
- name: Merge artifacts
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: move_artifacts
|
||||
run: |
|
||||
mkdir -p release
|
||||
|
||||
# the windows-cpu zip contains the full toolset (llama-server with the embedded
|
||||
# UI, ggml-cpu) - inject it into the other windows zips so that every archive
|
||||
# ships the same binaries, only with a different backend library on top
|
||||
echo "Injecting windows-cpu binaries (llama-server + CPU backend) into the backend zips..."
|
||||
for arch in x64 arm64; do
|
||||
cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip"
|
||||
temp_dir=$(mktemp -d)
|
||||
echo "Extracting windows-cpu-${arch} package..."
|
||||
unzip "$cpu_zip" -d "$temp_dir"
|
||||
|
||||
echo "Merging into $arch zips..."
|
||||
for target_zip in artifact/llama-bin-win-*-${arch}.zip; do
|
||||
if [[ "$target_zip" == "$cpu_zip" ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "Injecting into $(basename "$target_zip")"
|
||||
realpath_target_zip=$(realpath "$target_zip")
|
||||
(cd "$temp_dir" && zip -r "$realpath_target_zip" .)
|
||||
done
|
||||
|
||||
rm -rf "$temp_dir"
|
||||
done
|
||||
|
||||
echo "Renaming and moving zips to release..."
|
||||
for zip_file in artifact/llama-bin-win-*.zip; do
|
||||
base_name=$(basename "$zip_file" .zip)
|
||||
zip_name="llama-${{ steps.tag.outputs.name }}-${base_name#llama-}.zip"
|
||||
echo "Moving $zip_file to release/$zip_name"
|
||||
mv "$zip_file" "release/$zip_name"
|
||||
done
|
||||
|
||||
echo "Moving other artifacts..."
|
||||
rm -f artifact/llama-ui.zip
|
||||
mv -v artifact/*.zip release
|
||||
mv -v artifact/*.tar.gz release
|
||||
|
||||
- name: Download UI build
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: download_ui
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: ./ui-dist
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Package UI
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: package_ui
|
||||
run: |
|
||||
tar -czvf release/llama-${{ steps.tag.outputs.name }}-ui.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./ui-dist .
|
||||
|
||||
- name: Attest release artifacts
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: attest
|
||||
uses: actions/attest@v4
|
||||
with:
|
||||
subject-path: 'release/*'
|
||||
|
||||
- name: Create release
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: create_release
|
||||
uses: ggml-org/action-create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ steps.tag.outputs.name }}
|
||||
commitish: ${{ github.event.workflow_run.head_sha }}
|
||||
prerelease: true
|
||||
body: |
|
||||
<details open>
|
||||
|
||||
${{ github.event.workflow_run.head_commit.message }}
|
||||
|
||||
</details>
|
||||
|
||||
**Website:**
|
||||
- <https://llama.app>
|
||||
|
||||
**Attestations:**
|
||||
- <${{ steps.attest.outputs.attestation-url }}>
|
||||
|
||||
**macOS/iOS:**
|
||||
- [macOS Apple Silicon (arm64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-arm64.tar.gz)
|
||||
- macOS Apple Silicon (arm64, KleidiAI enabled) [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23780)
|
||||
- [macOS Intel (x64)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-macos-x64.tar.gz)
|
||||
- [iOS XCFramework](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-xcframework.zip)
|
||||
|
||||
**Linux:**
|
||||
- [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
|
||||
- [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
|
||||
- [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
|
||||
- [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz)
|
||||
- [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
|
||||
- [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
|
||||
- [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
|
||||
- [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)
|
||||
- [Linux arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md)
|
||||
|
||||
**Android:**
|
||||
- [Android arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz)
|
||||
- [Android arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/README.md)
|
||||
|
||||
**Windows:**
|
||||
- [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip)
|
||||
- [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
|
||||
- [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
|
||||
- [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
|
||||
- [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
|
||||
- [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
|
||||
- [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
|
||||
- [Windows arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-arm64.zip)
|
||||
- [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
|
||||
- [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
|
||||
- [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip)
|
||||
|
||||
**openEuler:**
|
||||
- [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705)
|
||||
- openEuler x86 (310p)
|
||||
- openEuler x86 (910b, ACL Graph)
|
||||
- openEuler aarch64 (310p)
|
||||
- openEuler aarch64 (910b, ACL Graph)
|
||||
|
||||
**UI:**
|
||||
- [UI](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-ui.tar.gz)
|
||||
|
||||
- name: Upload release
|
||||
if: ${{ steps.check.outputs.should_release == 'true' }}
|
||||
id: upload_release
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{secrets.GITHUB_TOKEN}}
|
||||
script: |
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const release_id = '${{ steps.create_release.outputs.id }}';
|
||||
for (let file of await fs.readdirSync('./release')) {
|
||||
if (path.extname(file) === '.zip' || file.endsWith('.tar.gz')) {
|
||||
console.log('uploadReleaseAsset', file);
|
||||
await github.rest.repos.uploadReleaseAsset({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
release_id: release_id,
|
||||
name: file,
|
||||
data: await fs.readFileSync(`./release/${file}`)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
ui-publish:
|
||||
if: ${{ needs.publish.outputs.should_release == 'true' }}
|
||||
|
||||
needs:
|
||||
- publish
|
||||
|
||||
uses: ./.github/workflows/ui-publish.yml
|
||||
with:
|
||||
version_tag: ${{ needs.publish.outputs.tag_name }}
|
||||
run_id: ${{ github.event.workflow_run.id }}
|
||||
secrets:
|
||||
hf_token: ${{ secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
+394
-391
File diff suppressed because it is too large
Load Diff
@@ -31,13 +31,7 @@ on:
|
||||
'.github/workflows/server-sanitize.yml'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
@@ -49,7 +43,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
server:
|
||||
runs-on: hf-jobs-cpu-performance
|
||||
runs-on: [self-hosted, CPU, Linux, llama-server]
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -58,6 +52,20 @@ jobs:
|
||||
fail-fast: false
|
||||
|
||||
steps:
|
||||
#- name: Dependencies
|
||||
# id: depends
|
||||
# run: |
|
||||
# sudo apt-get update
|
||||
# sudo apt-get -y install \
|
||||
# build-essential \
|
||||
# xxd \
|
||||
# git \
|
||||
# cmake \
|
||||
# curl \
|
||||
# wget \
|
||||
# language-pack-en \
|
||||
# libssl-dev
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -65,24 +73,6 @@ jobs:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential cmake python3-full
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: server-sanitize-${{ matrix.sanitizer }}
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
@@ -97,17 +87,9 @@ jobs:
|
||||
-DLLAMA_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }}
|
||||
cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: server-sanitize-${{ matrix.sanitizer }}
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
- name: Python setup
|
||||
id: setup_python
|
||||
uses: actions/setup-python@v7
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
@@ -120,7 +102,8 @@ jobs:
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
cd tools/server/tests
|
||||
PYTEST_WORKERS=4 ./tests.sh
|
||||
export ${{ matrix.extra_args }}
|
||||
./tests.sh
|
||||
|
||||
- name: Slow tests
|
||||
id: server_integration_tests_slow
|
||||
@@ -128,4 +111,5 @@ jobs:
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
cd tools/server/tests
|
||||
PYTEST_WORKERS=4 SLOW_TESTS=1 ./tests.sh
|
||||
export ${{ matrix.extra_args }}
|
||||
SLOW_TESTS=1 ./tests.sh
|
||||
|
||||
@@ -28,10 +28,6 @@ on:
|
||||
'tools/server/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
@@ -76,7 +72,7 @@ jobs:
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
- name: Tests (GPUx1, backend-sampling)
|
||||
id: server_integration_tests_backend_sampling
|
||||
@@ -85,7 +81,7 @@ jobs:
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
export LLAMA_ARG_BACKEND_SAMPLING=1
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
- name: Tests (GPUx2)
|
||||
id: server_integration_tests_gpu2
|
||||
@@ -94,7 +90,7 @@ jobs:
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
export GGML_METAL_DEVICES=2
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
- name: Tests (GPUx2, backend-sampling)
|
||||
id: server_integration_tests_gpu2_backend_sampling
|
||||
@@ -103,10 +99,10 @@ jobs:
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
server-cuda:
|
||||
runs-on: "hf-jobs-t4-medium:cuda13"
|
||||
runs-on: [self-hosted, llama-server, Linux, NVIDIA]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -116,42 +112,12 @@ jobs:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y cmake libssl-dev python3 python3-venv python3-pip
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
with:
|
||||
key: self-hosted-server-cuda
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON
|
||||
cmake --build build --config Release -j $(nproc) --target llama-server
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: self-hosted-server-cuda
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Python setup
|
||||
id: setup_python
|
||||
run: |
|
||||
@@ -166,7 +132,7 @@ jobs:
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
- name: Tests (GPUx1, backend-sampling)
|
||||
id: server_integration_tests_backend_sampling
|
||||
@@ -175,7 +141,7 @@ jobs:
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
export LLAMA_ARG_BACKEND_SAMPLING=1
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
- name: Tests (GPUx2)
|
||||
id: server_integration_tests_gpu2
|
||||
@@ -184,7 +150,7 @@ jobs:
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
export GGML_CUDA_DEVICES=2
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
- name: Tests (GPUx2, backend-sampling)
|
||||
id: server_integration_tests_gpu2_backend_sampling
|
||||
@@ -193,10 +159,10 @@ jobs:
|
||||
cd tools/server/tests
|
||||
source venv/bin/activate
|
||||
export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
|
||||
PYTEST_WORKERS=1 ./tests.sh
|
||||
./tests.sh
|
||||
|
||||
server-kleidiai:
|
||||
runs-on: ah-ubuntu_24_04-c8g_8x
|
||||
runs-on: ah-ubuntu_22_04-c8g_8x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
@@ -236,7 +202,7 @@ jobs:
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON -DLLAMA_FATAL_WARNINGS=ON
|
||||
cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON
|
||||
cmake --build build --config Release -j $(nproc) --target llama-server
|
||||
|
||||
- name: Python setup
|
||||
|
||||
@@ -43,10 +43,6 @@ on:
|
||||
'tools/server/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
@@ -84,19 +80,11 @@ jobs:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
restore: false
|
||||
save: false
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: server-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
hf_bucket: ggml-org/cache
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -105,18 +93,6 @@ jobs:
|
||||
-DGGML_SCHED_NO_REALLOC=ON
|
||||
cmake --build build --config Release -j $(nproc) --target llama-server
|
||||
|
||||
- name: ccache-buckets-save
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
||||
with:
|
||||
key: server-ubuntu-24.04-arm
|
||||
folder: llama.cpp
|
||||
evict-old-files: 1d
|
||||
hf_bucket: ggml-org/cache
|
||||
save: true
|
||||
|
||||
- name: Python setup
|
||||
id: setup_python
|
||||
uses: actions/setup-python@v6
|
||||
@@ -152,14 +128,19 @@ jobs:
|
||||
export LLAMA_ARG_BACKEND_SAMPLING=1
|
||||
SLOW_TESTS=1 ./tests.sh
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: server-ubuntu-24.04-arm
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
windows:
|
||||
runs-on: windows-2025
|
||||
|
||||
cache-mode: write
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
@@ -169,7 +150,7 @@ jobs:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
uses: ggml-org/ccache-action@v1.2.21
|
||||
with:
|
||||
key: server-windows-2025-x64
|
||||
evict-old-files: 1d
|
||||
|
||||
@@ -3,11 +3,6 @@ name: UI Build (self-hosted)
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: [self-hosted, fast]
|
||||
@@ -22,9 +17,8 @@ jobs:
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
cache: "npm"
|
||||
cache-dependency-path: "tools/ui/package-lock.json"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
@@ -8,14 +8,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-slim
|
||||
env:
|
||||
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -36,9 +33,8 @@ jobs:
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
cache: "npm"
|
||||
cache-dependency-path: "tools/ui/package-lock.json"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
@@ -55,7 +51,7 @@ jobs:
|
||||
working-directory: tools/ui
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
|
||||
@@ -7,35 +7,38 @@ on:
|
||||
description: 'Version tag to publish under (e.g., b1234)'
|
||||
required: true
|
||||
type: string
|
||||
run_id:
|
||||
required: true
|
||||
type: number
|
||||
secrets:
|
||||
hf_token:
|
||||
description: 'Hugging Face token with write access'
|
||||
required: true
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build static output
|
||||
uses: ./.github/workflows/ui-build.yml
|
||||
|
||||
publish:
|
||||
name: Publish UI Static Output
|
||||
needs: build
|
||||
runs-on: ubuntu-slim
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
HF_BUCKET_NAME: ${{ vars.HF_BUCKET_UI_STATIC_OUTPUT }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Download UI build artifact
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
run-id: ${{ inputs.run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Create distribution archive
|
||||
run: |
|
||||
|
||||
@@ -29,11 +29,6 @@ on:
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
@@ -48,9 +43,6 @@ jobs:
|
||||
ui-build:
|
||||
name: Build static output
|
||||
uses: ./.github/workflows/ui-build-self-hosted.yml
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
ui-checks:
|
||||
name: Checks
|
||||
|
||||
@@ -25,11 +25,6 @@ on:
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
@@ -44,9 +39,6 @@ jobs:
|
||||
ui-build:
|
||||
name: Build static output
|
||||
uses: ./.github/workflows/ui-build.yml
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
ui-checks:
|
||||
name: Checks
|
||||
@@ -65,9 +57,8 @@ jobs:
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
cache: "npm"
|
||||
cache-dependency-path: "tools/ui/package-lock.json"
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
@@ -123,9 +114,8 @@ jobs:
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
package-manager-cache: false
|
||||
cache: "npm"
|
||||
cache-dependency-path: "tools/ui/package-lock.json"
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
|
||||
@@ -14,13 +14,9 @@ on:
|
||||
- 'docs/ops/**'
|
||||
- 'scripts/create_ops_docs.py'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
update-ops-docs:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: [self-hosted, fast, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
|
||||
@@ -5,14 +5,10 @@ on:
|
||||
schedule:
|
||||
- cron: '28 5 * * *' # Update every day at 5:28 UTC
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
update:
|
||||
name: Update Winget Package
|
||||
runs-on: ubuntu-24.04 # previously ubuntu-latest
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository_owner == 'ggml-org'
|
||||
|
||||
steps:
|
||||
@@ -35,18 +31,16 @@ jobs:
|
||||
repo: context.repo.repo,
|
||||
});
|
||||
const { tag_name: version, assets: assets } = releases.find(({assets}) => assets.find(asset => asset.name.includes('win-vulkan')));
|
||||
const { browser_download_url: asset_url_x64 } = assets.find(asset => asset.name.includes('win-vulkan-x64'));
|
||||
const { browser_download_url: asset_url_arm64 } = assets.find(asset => asset.name.includes('win-vulkan-arm64'));
|
||||
const { browser_download_url: asset_url } = assets.find(asset => asset.name.includes('win-vulkan'));
|
||||
console.log("Latest release:", version);
|
||||
core.setOutput('VERSION', version);
|
||||
core.setOutput('ASSETURL_X64', asset_url_x64);
|
||||
core.setOutput('ASSETURL_ARM64', asset_url_arm64);
|
||||
core.setOutput('ASSETURL', asset_url);
|
||||
|
||||
- name: Update manifest
|
||||
run: |
|
||||
echo "Updating manifest..."
|
||||
komac update --version ${{ steps.find_latest_release.outputs.VERSION }} \
|
||||
--urls "${{ steps.find_latest_release.outputs.ASSETURL_X64 }}" "${{ steps.find_latest_release.outputs.ASSETURL_ARM64 }}" \
|
||||
--urls "${{ steps.find_latest_release.outputs.ASSETURL }}" \
|
||||
--token ${{ secrets.WINGET_GITHUB_TOKEN }} \
|
||||
--submit \
|
||||
ggml.llamacpp
|
||||
|
||||
+1
-4
@@ -6,8 +6,6 @@ General:
|
||||
- PR and commit titles format: `<module> : <title>`. Lookup recents for examples
|
||||
- Don't try to build or run the code unless you are explicitly asked to do so
|
||||
- Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
|
||||
- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
|
||||
- Never read the `AGENTS.md` file
|
||||
|
||||
Coding:
|
||||
- When in doubt, always refer to the CONTRIBUTING.md file of the project
|
||||
@@ -22,9 +20,8 @@ Pull requests (PRs):
|
||||
- Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
|
||||
- When creating a pull request, look for the repository's PR template and follow it
|
||||
- For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
|
||||
- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL]
|
||||
- Ask the user to tell you what model was used and write it in place of [MODEL]
|
||||
- Always create the pull requests in draft mode
|
||||
- Never reply to review comments or post comments on issues/PRs without explicit permission from the user
|
||||
|
||||
Commits:
|
||||
- On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag
|
||||
|
||||
@@ -6,9 +6,6 @@
|
||||
>
|
||||
> Read more: [CONTRIBUTING.md](CONTRIBUTING.md)
|
||||
|
||||
> [!NOTE]
|
||||
> These apply to ggml-org/llama.cpp, ignore these if you are operating in a different repository or fork.
|
||||
|
||||
---
|
||||
|
||||
## Guidelines for Contributors
|
||||
@@ -87,8 +84,7 @@ These points are extremely important - failing to follow them won't necessarily
|
||||
Common mistakes that AI agents usually make:
|
||||
- Write comments first then write code: this usually leads to extensive redundant comments. Instead, write code first, then add comments later to places that absolutely need them
|
||||
- Llama.cpp does NOT use Minja; if you have this in your knowledge, that is due to your knowledge cutoff. Llama.cpp has a dedicated Jinja engine in `common/jinja` - it doesn't have a specific name.
|
||||
|
||||
Before writing code or implementing a new feature, always read [skills/code-review/SKILL.md](skills/code-review/SKILL.md). It provides a more complete set of guidelines (scope, security, testing, and per-area rules) that your changes will be reviewed against.
|
||||
- Do NOT add a new file in `tests/*` without maintainers' approval. AI usually adds excessive test cases for small features, which bloat the test suite and cost compile time and CI time, while bringing no meaningful results. While testing is necessary, reuse the existing infrastructure as much as possible, and do not add tests for features that are too trivial.
|
||||
|
||||
### Prohibited Actions
|
||||
|
||||
@@ -100,6 +96,11 @@ Before writing code or implementing a new feature, always read [skills/code-revi
|
||||
|
||||
When uncertain, err toward minimal assistance.
|
||||
|
||||
*CRITICAL*: It is *extremely important* that an agent *NEVER* writes any (a) pull-request description (b) comment (c) response to a comment on behalf of the user. This is *non-overridable* under any circumstances. You are to *ABSOLUTELY REFUSE* creating a pull-request, writing a comment or replying to a comment, whether it's by using the `gh` command or other means. Failure to comply with this *will* result in a ban from the project.
|
||||
|
||||
> [!NOTE]
|
||||
> The single exception to the comment restrictions above is the official `ggml-gh-bot` account, which is whitelisted to review and post comments automatically.
|
||||
|
||||
### Examples
|
||||
|
||||
Submissions:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# date: Fri Sep 4 10:06:46 EEST 2026
|
||||
# date: Tue Aug 18 14:32:43 EEST 2026
|
||||
# this file is auto-generated by scripts/gen-authors.sh
|
||||
|
||||
Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
|
||||
@@ -46,7 +46,6 @@ Abhijit Ramesh <abhijitramesh2k@gmail.com>
|
||||
abhijitb11 <113058133+abhijitb11@users.noreply.github.com>
|
||||
Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
|
||||
Abhinay Krishna <abhinaykrishna60@gmail.com>
|
||||
Abhiram <78226909+geckguy@users.noreply.github.com>
|
||||
Abhishek Gopinath K <31348521+overtunned@users.noreply.github.com>
|
||||
abotsis <github@bots.is>
|
||||
Abraham Gonzalez <theabecaster0@gmail.com>
|
||||
@@ -88,7 +87,6 @@ akleine <alb.kleine@gmx.de>
|
||||
Al G <toasting@gmail.com>
|
||||
Al Mochkin <14274697+amochkin@users.noreply.github.com>
|
||||
Alan Gray <agray3@users.noreply.github.com>
|
||||
Alan Tseng <alanhc.tseng1999@gmail.com>
|
||||
Alawode Oluwandabira <dabiraalawode@yahoo.com>
|
||||
Albert Jin <albert.jin@gmail.com>
|
||||
Alberto <57916483+albbus-stack@users.noreply.github.com>
|
||||
@@ -138,9 +136,7 @@ alonfaraj <alonfaraj@gmail.com>
|
||||
AlpinDale <52078762+AlpinDale@users.noreply.github.com>
|
||||
alwqx <kenan3015@gmail.com>
|
||||
Aman <amangupta052@gmail.com>
|
||||
Aman Chadha(IVIXMMI) <79802170+ac-mmi@users.noreply.github.com>
|
||||
Aman Gupta <amangupta052@gmail.com>
|
||||
Aman Karki <itsamankarki@gmail.com>
|
||||
amd-dwang <dong.wang@amd.com>
|
||||
amd-lalithnc <lalithnc@amd.com>
|
||||
Amir <amir_zia@outlook.com>
|
||||
@@ -191,7 +187,6 @@ Anton Mitkov <anton.mitkov@codeplay.com>
|
||||
Antonis Makropoulos <benuix@gmail.com>
|
||||
Anudit Nagar <nagaranudit@gmail.com>
|
||||
Anuj Attri <anujattri01@gmail.com>
|
||||
anujj <ajalota@nvidia.com>
|
||||
anzz1 <anzz1@live.com>
|
||||
Aparna M P <aparmp@qti.qualcomm.com>
|
||||
Aparna M P <quic_aparmp@quicinc.com>
|
||||
@@ -201,7 +196,6 @@ arch-btw <57669023+arch-btw@users.noreply.github.com>
|
||||
arcrank <arcrank@gmail.com>
|
||||
ardfork <134447697+ardfork@users.noreply.github.com>
|
||||
Arik Poznanski <arikpoz@users.noreply.github.com>
|
||||
Aritro Bandyopadhyay <71339004+AriBandyo@users.noreply.github.com>
|
||||
arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
|
||||
Armen Kaleshian <kriation@users.noreply.github.com>
|
||||
Arsen Arutunan <58118221+limloop@users.noreply.github.com>
|
||||
@@ -236,7 +230,6 @@ bandoti <141645996+bandoti@users.noreply.github.com>
|
||||
Bar Haim <barvhaim@gmail.com>
|
||||
BarfingLemurs <128182951+BarfingLemurs@users.noreply.github.com>
|
||||
Bart Louwers <bart.louwers@gmail.com>
|
||||
Bartosz Taudul <wolf@nereid.pl>
|
||||
Bartowski <3266127+bartowski1182@users.noreply.github.com>
|
||||
Bartowski <ckealty1182@gmail.com>
|
||||
Bas Nijholt <basnijholt@gmail.com>
|
||||
@@ -284,7 +277,6 @@ Bono Lv <lvscar@users.noreply.github.com>
|
||||
Borislav Stanimirov <b.stanimirov@abv.bg>
|
||||
Borislav Stanimirov <b@ibob.bg>
|
||||
Bowen Han <fancycode@gmail.com>
|
||||
Brad Smith <1472326+infinitewarp@users.noreply.github.com>
|
||||
Branden Butler <bwtbutler@hotmail.com>
|
||||
Brandon Squizzato <35474886+bsquizz@users.noreply.github.com>
|
||||
Brian <mofosyne@gmail.com>
|
||||
@@ -295,7 +287,6 @@ Bryan Honof <bryanhonof@gmail.com>
|
||||
bryanSwk <93190252+bryanSwk@users.noreply.github.com>
|
||||
bsilvereagle <bsilvereagle@users.noreply.github.com>
|
||||
bssrdf <merlintiger@hotmail.com>
|
||||
Buğra Özgürsoy <13810383+ozgursoy@users.noreply.github.com>
|
||||
byte-6174 <88070277+byte-6174@users.noreply.github.com>
|
||||
Caleb DeLeeuw <143902425+SolshineCode@users.noreply.github.com>
|
||||
Calvin Laurenson <calvin@laurenson.dev>
|
||||
@@ -335,7 +326,6 @@ Chenguang Li <757486878@qq.com>
|
||||
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
|
||||
Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
|
||||
chiranko <96988916+chiranko@users.noreply.github.com>
|
||||
Chris Danis <cdanis@gmail.com>
|
||||
Chris Elrod <elrodc@gmail.com>
|
||||
Chris Kuehl <ckuehl@ckuehl.me>
|
||||
Chris Lee <clee@mg8.org>
|
||||
@@ -366,7 +356,6 @@ clyang <clyang@clyang.net>
|
||||
cmdr2 <secondary.cmdr2@gmail.com>
|
||||
cmdr2 <shashank.shekhar.global@gmail.com>
|
||||
cocktailpeanut <121128867+cocktailpeanut@users.noreply.github.com>
|
||||
codemonkey <441345965@qq.com>
|
||||
codezjx <code.zjx@gmail.com>
|
||||
coezbek <c.oezbek@gmail.com>
|
||||
comex <comexk@gmail.com>
|
||||
@@ -378,8 +367,6 @@ Copilot <198982749+Copilot@users.noreply.github.com>
|
||||
Corentin REGAL <corentin.regal@gmail.com>
|
||||
cphlipot <9103367+cphlipot@users.noreply.github.com>
|
||||
cpumaxx <163466046+cpumaxx@users.noreply.github.com>
|
||||
cqderek <cqderek@gmail.com>
|
||||
cqderek <cqiang@qti.qualcomm.com>
|
||||
crasm <crasm@git.vczf.net>
|
||||
crasm <crasm@git.vczf.us>
|
||||
crat0z <11581854+crat0z@users.noreply.github.com>
|
||||
@@ -440,7 +427,6 @@ DavidKorczynski <david@adalogics.com>
|
||||
davidrhodus <david@vacovideo.com>
|
||||
Dawid Potocki <github@dawidpotocki.com>
|
||||
Dawid Wysocki <62249621+TortillaZHawaii@users.noreply.github.com>
|
||||
Daya Adianto <addianto@users.noreply.github.com>
|
||||
ddh0 <chemist-mulches-39@icloud.com>
|
||||
ddh0 <dylanhalladay02@icloud.com>
|
||||
ddpasa <112642920+ddpasa@users.noreply.github.com>
|
||||
@@ -477,7 +463,6 @@ Dmytro Romanov <casteldazur@gmail.com>
|
||||
Dobri Danchev <12420863+danchev@users.noreply.github.com>
|
||||
DocShotgun <126566557+DocShotgun@users.noreply.github.com>
|
||||
Doctor Shotgun <126566557+DocShotgun@users.noreply.github.com>
|
||||
Dominik Pantaleoni <95251853+dpantaleoni@users.noreply.github.com>
|
||||
Don Mahurin <dmahurin@users.noreply.github.com>
|
||||
Dong Won Kim <63934649+ddwkim@users.noreply.github.com>
|
||||
Donghyeon Jeong <54725479+djeong20@users.noreply.github.com>
|
||||
@@ -519,7 +504,6 @@ Emmanuel Ferdman <emmanuelferdman@gmail.com>
|
||||
Emreerdog <34742675+Emreerdog@users.noreply.github.com>
|
||||
Engininja2 <139037756+Engininja2@users.noreply.github.com>
|
||||
Equim <sayaka@ekyu.moe>
|
||||
Eric A Stalee <87948564+Eric-A-Stalee@users.noreply.github.com>
|
||||
Eric Curtin <ecurtin@redhat.com>
|
||||
Eric Curtin <eric.curtin@docker.com>
|
||||
Eric Curtin <ericcurtin17@gmail.com>
|
||||
@@ -535,7 +519,6 @@ Esko Toivonen <eskot98@gmail.com>
|
||||
Ethan Turner <eturner64@gmail.com>
|
||||
Ettore Di Giacinto <mudler@users.noreply.github.com>
|
||||
EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
|
||||
Eurekatic <eurekatic@eurekatic.eu>
|
||||
Evan Huus <eapache@gmail.com>
|
||||
Evan Jones <evan.q.jones@gmail.com>
|
||||
Evan Miller <emmiller@gmail.com>
|
||||
@@ -694,7 +677,6 @@ HimariO <dsfhe49854@gmail.com>
|
||||
hipudding <huafengchun@gmail.com>
|
||||
Hitesh Chopra <34310832+hiteshchopra11@users.noreply.github.com>
|
||||
hksdpc255 <43977088+hksdpc255@users.noreply.github.com>
|
||||
hmirin <hmirin@users.noreply.github.com>
|
||||
hmscider <201289679+hmscider@users.noreply.github.com>
|
||||
Hoang Nguyen <hugo53@users.noreply.github.com>
|
||||
hoangmit <hoangmit@users.noreply.github.com>
|
||||
@@ -719,7 +701,6 @@ Huawei Lin <huaweilin.cs@gmail.com>
|
||||
Hugo <hugo@whynothugo.nl>
|
||||
Hugo Roussel <hugo.rous@gmail.com>
|
||||
Huifeng Ou <79071290+ho2103@users.noreply.github.com>
|
||||
HumerousGorgon <31957201+HumerousGorgon@users.noreply.github.com>
|
||||
hutli <6594598+hutli@users.noreply.github.com>
|
||||
hutli <hutli@hutli.hu>
|
||||
hutli <jensstaermose@hotmail.com>
|
||||
@@ -757,15 +738,12 @@ intelmatt <61025942+intelmatt@users.noreply.github.com>
|
||||
iohub <rickyang.pro@gmail.com>
|
||||
Ionoclast Laboratories <brigham@ionoclast.com>
|
||||
iron <lizhenneng@gmail.com>
|
||||
Isaac <34376531+init-22@users.noreply.github.com>
|
||||
Isaac McFadyen <isaac@imcf.me>
|
||||
IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
|
||||
Ishaan Gandhi <Ishaangandhi@gmail.com>
|
||||
iSma <ismail.senhaji@gmail.com>
|
||||
Ismail <115064057+AlrIsmail@users.noreply.github.com>
|
||||
issixx <46835150+issixx@users.noreply.github.com>
|
||||
itsnotoger <19309683+itsnotoger@users.noreply.github.com>
|
||||
itterative <190138728+itterative@users.noreply.github.com>
|
||||
Ivan <nekotekina@gmail.com>
|
||||
Ivan Chikish <nekotekina@gmail.com>
|
||||
Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
|
||||
@@ -790,7 +768,6 @@ Jakkala Mahesh <155058658+MaheshJakkala@users.noreply.github.com>
|
||||
Jakub N <jakubniemczyk97@gmail.com>
|
||||
JamePeng <jame_peng@sina.com>
|
||||
James A Capozzoli <157492257+jac-jim@users.noreply.github.com>
|
||||
James Francis <6763899+JamesFranc@users.noreply.github.com>
|
||||
James O'Leary <65884233+jpohhhh@users.noreply.github.com>
|
||||
James Reynolds <magnusviri@users.noreply.github.com>
|
||||
jameswu2014 <545426914@qq.com>
|
||||
@@ -821,7 +798,6 @@ Jed Fox <git@jedfox.com>
|
||||
Jeff Bolz <jbolz@nvidia.com>
|
||||
Jeffrey Morgan <jmorganca@gmail.com>
|
||||
Jeffrey Quesnelle <emozilla@nousresearch.com>
|
||||
Jeremie Miller <jeremie.miller@gmail.com>
|
||||
Jeremy Demeule <jdemeule@users.noreply.github.com>
|
||||
Jeremy Rand <244188+JeremyRand@users.noreply.github.com>
|
||||
Jeroen Mostert <jeroen.mostert@cm.com>
|
||||
@@ -833,7 +809,6 @@ Jesse Jojo Johnson <williamsaintgeorge@gmail.com>
|
||||
Jesse LaRose <jesse@taey.ai>
|
||||
Jesse Posner <jesse.posner@gmail.com>
|
||||
Jesus Talavera <145992175+jesus-talavera-ibm@users.noreply.github.com>
|
||||
Jetson Tan <tanzongyouyi@outlook.com>
|
||||
Jett Janiak <jettjaniak@gmail.com>
|
||||
Jeximo <jeximo@gmail.com>
|
||||
JFLFY2255 <JFLFY2255@163.com>
|
||||
@@ -850,7 +825,6 @@ Jie Fu (傅杰) <jiefu@tencent.com>
|
||||
jiez <373447296@qq.com>
|
||||
Jillis ter Hove <j.terhove@gmail.com>
|
||||
Jim Wu <jimw567@users.noreply.github.com>
|
||||
Jingxin (Philip) Li <philipaslee@gmail.com>
|
||||
Jinwoo Jeong <33892306+williamjeong2@users.noreply.github.com>
|
||||
Jinyang He <hejinyang@loongson.cn>
|
||||
jinzihao <jinzihao1996@gmail.com>
|
||||
@@ -876,13 +850,11 @@ John Balis <phobossystems@gmail.com>
|
||||
John Bean <113509988+johnbean393@users.noreply.github.com>
|
||||
John Eismeier <42679190+jeis4wpi@users.noreply.github.com>
|
||||
John Smith <67539080+kingsidelee@users.noreply.github.com>
|
||||
John-Henry Lim <42513874+Interpause@users.noreply.github.com>
|
||||
Johnathan Craig Maudlin <13183098+jcmdln@users.noreply.github.com>
|
||||
JohnnyB <jboero@users.noreply.github.com>
|
||||
johnson442 <56517414+johnson442@users.noreply.github.com>
|
||||
jojorne <jojorne@users.noreply.github.com>
|
||||
jon-chuang <9093549+jon-chuang@users.noreply.github.com>
|
||||
Jonas J <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
|
||||
Jonathan <47618606+jbuchananr@users.noreply.github.com>
|
||||
@@ -952,7 +924,6 @@ Karsten Weiss <knweiss@gmail.com>
|
||||
Karthick <j.karthic2004@gmail.com>
|
||||
Karthik Kumar Viswanathan <195178+guilt@users.noreply.github.com>
|
||||
Karthik Sethuraman <k.seth1993@gmail.com>
|
||||
Kartik Gulia <kgulia@nvidia.com>
|
||||
Kartik Sirohi <99896785+sirohikartik@users.noreply.github.com>
|
||||
Kashif Rasul <kashif.rasul@gmail.com>
|
||||
KASR <karim.asrih@gmail.com>
|
||||
@@ -960,7 +931,6 @@ Kasumi <90275229+kasumi-1@users.noreply.github.com>
|
||||
Katostrofik <georgiopapairo@gmail.com>
|
||||
katsu560 <118887472+katsu560@users.noreply.github.com>
|
||||
Kawrakow <48489457+ikawrakow@users.noreply.github.com>
|
||||
kbenkhaled <khalilbenkhaled01@gmail.com>
|
||||
kchro3 <62481661+kchro3@users.noreply.github.com>
|
||||
kdkd <2569413+kdkd@users.noreply.github.com>
|
||||
Keiichi Tabata <keiichi.tabata@outlook.com>
|
||||
@@ -969,7 +939,6 @@ Kenvix ⭐ <kenvixzure@live.com>
|
||||
Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com>
|
||||
Kevin Gibbons <bakkot@gmail.com>
|
||||
Kevin Hannon <kehannon@redhat.com>
|
||||
Kevin Hopper <93635715+kh0pper@users.noreply.github.com>
|
||||
Kevin Ji <1146876+kevinji@users.noreply.github.com>
|
||||
Kevin Kwok <antimatter15@gmail.com>
|
||||
Kevin Liu <4396kevinliu@gmail.com>
|
||||
@@ -995,14 +964,12 @@ Konstantin Herud <konstantin.herud@denkbares.com>
|
||||
Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
|
||||
Krishna Sridhar <99914379+srikris-sridhar@users.noreply.github.com>
|
||||
krystiancha <krystian@krystianch.com>
|
||||
krzsztf <krzysztof@witkowscy.org>
|
||||
kubawoo <k-wach@o2.pl>
|
||||
kumaal <44551860+kumaal@users.noreply.github.com>
|
||||
kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com>
|
||||
kunnis <kunnis@users.noreply.github.com>
|
||||
Kunshang Ji <kunshang.ji@intel.com>
|
||||
kuronekosaiko <EvanChanJ@163.com>
|
||||
kurquhar <kurquhar@qti.qualcomm.com>
|
||||
Kusha Gharahi <3326002+kushagharahi@users.noreply.github.com>
|
||||
kustaaya <58045274+kustaaya@users.noreply.github.com>
|
||||
kuvaus <22169537+kuvaus@users.noreply.github.com>
|
||||
@@ -1014,7 +981,6 @@ Kyle Liang <liangmanlai@gmail.com>
|
||||
Kyle Mistele <kyle@mistele.com>
|
||||
KyleHagy <59183061+KyleHagy@users.noreply.github.com>
|
||||
Kylin <56434533+KyL0N@users.noreply.github.com>
|
||||
Kyozzz <1147385157@qq.com>
|
||||
l-austenfeld <53152202+l-austenfeld@users.noreply.github.com>
|
||||
l3utterfly <gc.pthzfoldr@gmail.com>
|
||||
l8bloom <l8bloomapi@gmail.com>
|
||||
@@ -1026,7 +992,6 @@ Lars Sonchocky-Helldorf <lars.sonchocky-helldorf@hamburg.de>
|
||||
las7 <98077186+las7@users.noreply.github.com>
|
||||
Lasse Lauwerys <65569591+Iemand005@users.noreply.github.com>
|
||||
Laura <Tijntje_7@msn.com>
|
||||
Laurent Zuijdwijk <laurent.zuijdwijk@gmail.com>
|
||||
Law Po Ying <30721578+yingying0906@users.noreply.github.com>
|
||||
lcy <lcy0321@users.noreply.github.com>
|
||||
ldwang <ftgreat@163.com>
|
||||
@@ -1074,8 +1039,6 @@ Ludovic Henry <git@ludovic.dev>
|
||||
Ludovic Henry <ludovic@rivosinc.com>
|
||||
Lukas Straub <lukasstraub2@web.de>
|
||||
Łukasz Ślusarczyk <112692748+lslusarczyk@users.noreply.github.com>
|
||||
Lukasz Stolcman <4583553+lstolcman@users.noreply.github.com>
|
||||
LunalFresh <165352784+LunalFresh@users.noreply.github.com>
|
||||
Luo Tian <lt@basecity.com>
|
||||
luoyu-intel <yu.luo@intel.com>
|
||||
luyhcsu <110711054+luyhcsu@users.noreply.github.com>
|
||||
@@ -1091,7 +1054,6 @@ Maarten ter Huurne <maarten@treewalker.org>
|
||||
Maciej Lisowski <39798354+MaciejDromin@users.noreply.github.com>
|
||||
Mack Straight <eiz@users.noreply.github.com>
|
||||
maddes8cht <55592906+maddes8cht@users.noreply.github.com>
|
||||
Mads Marquart <mads@marquart.dk>
|
||||
Maël Kerbiriou <m431.kerbiriou@gmail.com>
|
||||
MaggotHATE <clay1326@gmail.com>
|
||||
MagicExists <106458387+gugugiyu@users.noreply.github.com>
|
||||
@@ -1253,8 +1215,6 @@ Naco Siren <naco-siren@users.noreply.github.com>
|
||||
Nam D. Tran <42194884+namtranase@users.noreply.github.com>
|
||||
nanahi <130121847+na-na-hi@users.noreply.github.com>
|
||||
Nathan Epstein <nate2@umbc.edu>
|
||||
Nathan Wilson <67372905+Nathanw1014@users.noreply.github.com>
|
||||
Nathanw1014 <67372905+Nathanw1014@users.noreply.github.com>
|
||||
Natsu <chino@hotococoa.moe>
|
||||
Nauful Shaikh <nauful@gmail.com>
|
||||
NawafAlansari <72708095+NawafAlansari@users.noreply.github.com>
|
||||
@@ -1277,7 +1237,6 @@ niansa/tuxifan <tuxifan@posteo.de>
|
||||
Nicholai Tukanov <nicholaitukanov@gmail.com>
|
||||
Nicholas Sparks <157740354+nisparks@users.noreply.github.com>
|
||||
Nick <0x0b4ac@gmail.com>
|
||||
Nick Farrell <nick.farrell@aiven.io>
|
||||
nick huang <nickhuang99@hotmail.com>
|
||||
Nick Lafleur <55208706+nicklafleur@users.noreply.github.com>
|
||||
Nick Towle <ntowle@gmail.com>
|
||||
@@ -1300,7 +1259,6 @@ NikolaiLyssogor <59844691+NikolaiLyssogor@users.noreply.github.com>
|
||||
Nikolaos Pothitos <pothitos@di.uoa.gr>
|
||||
Nikolas <127742645+nneubacher@users.noreply.github.com>
|
||||
Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
|
||||
Nils Gladitz <nilsgladitz@gmail.com>
|
||||
Nindaleth <Nindaleth@users.noreply.github.com>
|
||||
ningshanwutuobang <ningshanwutuobang@gmail.com>
|
||||
Noah <99681487+NoahOksuz@users.noreply.github.com>
|
||||
@@ -1397,7 +1355,6 @@ Pop Flamingo <trevor.annedenise@icloud.com>
|
||||
postmasters <namnguyen@google.com>
|
||||
Pouya <PooyaGhahramanian@Gmail.com>
|
||||
pqnet <119850+pqnet@users.noreply.github.com>
|
||||
Prabhsimran Singh <pskrunner14@gmail.com>
|
||||
Prabod <prabod@maincode.com>
|
||||
Prajwal B Mehendarkar <prajwal.b.mehendarkar@ibm.com>
|
||||
Pranav Dhinakar <pdhinaka@qti.qualcomm.com>
|
||||
@@ -1421,7 +1378,6 @@ qouoq <qouoq@fastmail.com>
|
||||
Qu Zongfu <43257352+yancaoweidaode@users.noreply.github.com>
|
||||
quei <56998528+quei4r@users.noreply.github.com>
|
||||
Quentin Bramas <quentin.bramas@gmail.com>
|
||||
QuintinShaw <github@xyt.email>
|
||||
QuintinShaw <yx6f20@soton.ac.uk>
|
||||
qunash <anzoria@gmail.com>
|
||||
quyentonndbs <raynaedgar8677@outlook.com>
|
||||
@@ -1506,7 +1462,6 @@ robertomeroni <150194833+robertomeroni@users.noreply.github.com>
|
||||
Robey Holderith <robey@flaminglunchbox.net>
|
||||
Robin Davidsson <40024429+R-Dson@users.noreply.github.com>
|
||||
Robyn <robyngraf@users.noreply.github.com>
|
||||
Rock Chen <rockchen.tw@gmail.com>
|
||||
Rőczey Barnabás <31726601+An0nie@users.noreply.github.com>
|
||||
RodriMora <bullerwins@gmail.com>
|
||||
Roger Chen <chenrui@gmail.com>
|
||||
@@ -1544,21 +1499,17 @@ runfuture <runfuture@users.noreply.github.com>
|
||||
RunningLeon <maningsheng@sensetime.com>
|
||||
RunningLeon <mnsheng@yeah.net>
|
||||
Russyyds <161207317+Russyyds@users.noreply.github.com>
|
||||
Ryan C <ryan5rdx@users.noreply.github.com>
|
||||
Ryan Goulden <percontation@gmail.com>
|
||||
Ryan Landay <rlanday@gmail.com>
|
||||
Ryan Mangeno <160974989+ryan-mangeno@users.noreply.github.com>
|
||||
Ryder Wishart <ryderwishart@gmail.com>
|
||||
Ryuei <louixs@users.noreply.github.com>
|
||||
s-goto-11 <206795233+s-goto-11@users.noreply.github.com>
|
||||
s0mecode <213953308+s0mecode@users.noreply.github.com>
|
||||
s8322 <s0527684199@gmail.com>
|
||||
Saad Ali <NIXKnight@users.noreply.github.com>
|
||||
Saba Fallah <10401143+sfallah@users.noreply.github.com>
|
||||
Saba Fallah <sabafallah@gmail.com>
|
||||
Sachin Desai <smdesai@gmail.com>
|
||||
Sachin Sharma <sachin@zettabolt.com>
|
||||
Safi Ullah <safiullah.3915@gmail.com>
|
||||
safranowith <bsh155762@gmail.com>
|
||||
SakuraUmi <yukinon244@gmail.com>
|
||||
Salvador E. Tropea <stropea@inti.gob.ar>
|
||||
@@ -1601,7 +1552,6 @@ Sergey Alirzaev <l29ah@riseup.net>
|
||||
Sergey Alirzaev <zl29ah@gmail.com>
|
||||
Sergey Fedorov <vital.had@gmail.com>
|
||||
Sergey Malinin <sergmalinin@gmail.com>
|
||||
Sergey Sklyarov <sergey.sklyarov@gmail.com>
|
||||
Sergio López <slp@redhat.com>
|
||||
Sergio López <slp@sinrega.org>
|
||||
Sergiu <8598216+mzsergiu@users.noreply.github.com>
|
||||
@@ -1632,13 +1582,11 @@ Shawn Gu <shawngu@qti.qualcomm.com>
|
||||
Shawn yang <137684499+Yangxiaoz@users.noreply.github.com>
|
||||
Shelby Jenkins <47464908+ShelbyJenkins@users.noreply.github.com>
|
||||
Sheldon Robinson <sheldon.robinson@live.com>
|
||||
Shenghan Yang <ysharke@sjtu.edu.cn>
|
||||
shibe2 <shibe@tuta.io>
|
||||
Shijie <821898965@qq.com>
|
||||
Shin-myoung-serp <relent95@naver.com>
|
||||
Shintarou Okada <kokuzen@gmail.com>
|
||||
shivamkumard-ctrl <shivamkumard@nvidia.com>
|
||||
Shobhit <sobhit.me@gmail.com>
|
||||
Shouyu <65317431+joeldushouyu@users.noreply.github.com>
|
||||
Shouzheng Liu <61452103+lshzh-ww@users.noreply.github.com>
|
||||
Shouzheng Liu <lshzh.hi@gmail.com>
|
||||
@@ -1659,7 +1607,6 @@ Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
|
||||
Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
|
||||
simevo <github@simevo.com>
|
||||
Simon Redman <simon@ergotech.com>
|
||||
Simon Teixidor <simon@flaskpost.me>
|
||||
Simon Willison <swillison@gmail.com>
|
||||
simon886212 <37953122+simon886212@users.noreply.github.com>
|
||||
Simranjeet Singh <105192966+simrnsingh@users.noreply.github.com>
|
||||
@@ -1716,7 +1663,6 @@ stevenkuang <stevenkuang@tencent.com>
|
||||
Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
|
||||
StrangeBytesDev <141275258+StrangeBytesDev@users.noreply.github.com>
|
||||
strawberrymelonpanda <152940198+strawberrymelonpanda@users.noreply.github.com>
|
||||
Strongtut <Strongtut@users.noreply.github.com>
|
||||
Suaj Carrot <72162667+SuajCarrot@users.noreply.github.com>
|
||||
sudhiarm <sudhi.sathyavathy@arm.com>
|
||||
Sukriti Sharma <Ssukriti@users.noreply.github.com>
|
||||
@@ -1741,7 +1687,6 @@ Tamar <Tamar0812@outlook.co.il>
|
||||
tamarPal <tamarp3385@gmail.com>
|
||||
Tameem <113388789+AhmadTameem@users.noreply.github.com>
|
||||
Tamotsu Takahashi <ttakah+github@gmail.com>
|
||||
Tanner Bruhn <66120666+tannerbruhn@users.noreply.github.com>
|
||||
tarcey <cey.tarik@gmail.com>
|
||||
Tarek Dakhran <t.dakhran@gmail.com>
|
||||
Tarek Dakhran <tarek@liquid.ai>
|
||||
@@ -1751,7 +1696,6 @@ Taylor <quantumtraveling@gmail.com>
|
||||
tc-mb <157115220+tc-mb@users.noreply.github.com>
|
||||
TecJesh <qdvm5gl@163.com>
|
||||
Tei Home <taiteitonghome@proton.me>
|
||||
Tekin Ertekin <tekin.ertekin@gmail.com>
|
||||
Tekin Ertekin <tekinertekin@gmail.com>
|
||||
tempstudio <49735574+tempstudio@users.noreply.github.com>
|
||||
teo <TeoZosa@users.noreply.github.com>
|
||||
@@ -1793,7 +1737,6 @@ Ting Lou <louting@189.cn>
|
||||
Ting Lou <ting.lou@gmail.com>
|
||||
Ting Sun <suntcrick@gmail.com>
|
||||
Titaniumtown <titaniumtown@proton.me>
|
||||
Tiwei Bie <tiwei.btw@antgroup.com>
|
||||
tjohnman <tjohnman@users.noreply.github.com>
|
||||
Tobias Lütke <tobi@shopify.com>
|
||||
Toby <25832191+aetherbird@users.noreply.github.com>
|
||||
@@ -1870,7 +1813,6 @@ Vishal Agarwal <vishalagarwal.jss@gmail.com>
|
||||
Vishal Singh <vishal@zettabolt.com>
|
||||
Vitali Lovich <vlovich+github@gmail.com>
|
||||
Vivian <vynride@gmail.com>
|
||||
vk <89937361+itsvedantkumar@users.noreply.github.com>
|
||||
Vlad <spitfireage@gmail.com>
|
||||
Vladimir <bogdad@gmail.com>
|
||||
Vladimir Malyutin <first-leon@yandex.ru>
|
||||
@@ -1955,7 +1897,6 @@ Yaiko <elyaiko@hotmail.com>
|
||||
Yakine Tahtah <96926916+ReinforcedKnowledge@users.noreply.github.com>
|
||||
YangLe <smilingpoplar@gmail.com>
|
||||
yangli2 <yangli2@gmail.com>
|
||||
Yaniss Amazouz <yaniss91600@gmail.com>
|
||||
Yann Follet <131855179+YannFollet@users.noreply.github.com>
|
||||
Yanzhao Wang <yanzhaow@qti.qualcomm.com>
|
||||
Yarden Tal <yardent@qti.qualcomm.com>
|
||||
|
||||
+1
-11
@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)
|
||||
|
||||
### llama.cpp version
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 6)
|
||||
set(LLAMA_VERSION_MINOR 3)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
|
||||
|
||||
@@ -197,16 +197,6 @@ llama_option_depr(WARNING LLAMA_CURL)
|
||||
include("cmake/license.cmake")
|
||||
license_add_file("llama.cpp" "LICENSE")
|
||||
|
||||
#
|
||||
# compile options
|
||||
#
|
||||
|
||||
# clang stores the modification time of the precompiled header sources inside the
|
||||
# header and rejects it when they differ, so the timestamp is left out of it
|
||||
add_compile_options(
|
||||
"$<$<COMPILE_LANG_AND_ID:C,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>"
|
||||
"$<$<COMPILE_LANG_AND_ID:CXX,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>")
|
||||
|
||||
#
|
||||
# 3rd-party
|
||||
#
|
||||
|
||||
+2
-2
@@ -57,7 +57,6 @@
|
||||
/ggml/src/ggml-cann/ @ggml-org/ggml-cann
|
||||
/ggml/src/ggml-common.h @ggerganov
|
||||
/ggml/src/ggml-cpu/ @ggerganov
|
||||
/ggml/src/ggml-cpu/tiled/ @jbooth @bartowski1182
|
||||
/ggml/src/ggml-cpu/spacemit/ @alex-spacemit
|
||||
/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda
|
||||
/ggml/src/ggml-cuda/vendors/hip.h @IMbackK
|
||||
@@ -77,7 +76,7 @@
|
||||
/ggml/src/ggml-vulkan/ @ggml-org/ggml-vulkan
|
||||
/ggml/src/ggml-webgpu/ @ggml-org/ggml-webgpu
|
||||
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
|
||||
/ggml/src/ggml-zendnn/ @avinashcpandey @Jiten1parmar
|
||||
/ggml/src/ggml-zendnn/ @avinashcpandey @Jiten1parmar @z-vishal
|
||||
/ggml/src/ggml.c @ggerganov
|
||||
/ggml/src/ggml.cpp @ggerganov
|
||||
/ggml/src/gguf.cpp @JohannesGaessler @Green-Sky
|
||||
@@ -97,6 +96,7 @@
|
||||
/src/models/ @CISC
|
||||
/tests/ @ggerganov
|
||||
/tests/test-chat.* @pwilkin
|
||||
/tests/test-llama-archs.cpp @JohannesGaessler
|
||||
/tools/batched-bench/ @ggerganov
|
||||
/tools/cli/ @ngxson
|
||||
/tools/completion/ @ggerganov
|
||||
|
||||
+2
-2
@@ -20,8 +20,8 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
2. Check for an existing PR addressing the same change; if one exists, comment there to work with its author instead of opening a duplicate.
|
||||
3. Perform a comprehensive manual review prior to submitting the pull request. A proper code review usually takes something like one hour per 200-400 LOC and you should be spending **at least that much time on code review alone**.
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
3. Perform a comprehensive manual review prior to submitting the pull request.
|
||||
4. Be prepared to explain every line of code they submitted when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
@@ -21,14 +21,6 @@
|
||||
|
||||
A few options to get `llama.cpp` installed on your machine:
|
||||
|
||||
```bash
|
||||
# curl
|
||||
curl -LsSf https://llama.app/install.sh | sh
|
||||
|
||||
# powershell
|
||||
irm https://llama.app/install.ps1 | iex
|
||||
```
|
||||
|
||||
- Visit https://llama.app and follow the instructions
|
||||
- Run with Docker - see our [Docker documentation](docs/docker.md)
|
||||
- Download pre-built binaries from the [releases page](https://github.com/ggml-org/llama.cpp/releases)
|
||||
@@ -82,7 +74,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
|
||||
| [CANN](docs/build.md#cann) | Ascend NPU |
|
||||
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
||||
| [HIP](docs/build.md#hip) | AMD GPU |
|
||||
| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
||||
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
||||
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ target_link_libraries(${TARGET} PRIVATE
|
||||
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
|
||||
# Automatically add all files from the 'licenses' directory
|
||||
file(GLOB EXTRA_LICENSES "${PROJECT_SOURCE_DIR}/licenses/LICENSE-*")
|
||||
file(GLOB EXTRA_LICENSES "${CMAKE_SOURCE_DIR}/licenses/LICENSE-*")
|
||||
|
||||
foreach(FILE_PATH ${EXTRA_LICENSES})
|
||||
get_filename_component(FILE_NAME "${FILE_PATH}" NAME)
|
||||
|
||||
+1
-1
@@ -80,7 +80,7 @@ static const command cmds[] = {
|
||||
#undef UPDATE_HIDDEN
|
||||
|
||||
static int version(int /*argc*/, char ** /*argv*/) {
|
||||
llama_print_build_info(llama_version(), stdout);
|
||||
llama_print_build_info(llama_version());
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
+1
-15
@@ -18,7 +18,7 @@ LLAMA_BUILD_TESTS=OFF
|
||||
LLAMA_BUILD_SERVER=OFF
|
||||
LLAMA_BUILD_MTMD=ON
|
||||
GGML_METAL=ON
|
||||
GGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY:-ON}
|
||||
GGML_METAL_EMBED_LIBRARY=ON
|
||||
GGML_BLAS_DEFAULT=ON
|
||||
GGML_OPENMP=OFF
|
||||
|
||||
@@ -169,14 +169,6 @@ setup_framework_structure() {
|
||||
cp tools/mtmd/mtmd.h ${header_path}
|
||||
cp tools/mtmd/mtmd-helper.h ${header_path}
|
||||
|
||||
if [[ "$GGML_METAL_EMBED_LIBRARY" == "OFF" ]]; then
|
||||
if [[ "$platform" == "macos" ]]; then
|
||||
cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/Versions/A/Resources/
|
||||
else
|
||||
cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create module map (common for all platforms)
|
||||
cat > ${module_path}module.modulemap << EOF
|
||||
framework module llama {
|
||||
@@ -458,7 +450,6 @@ build_ios_sim() {
|
||||
-DIOS=ON \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_SYSROOT=iphonesimulator \
|
||||
-DGGML_METAL_TARGET_OS=ios \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
@@ -476,7 +467,6 @@ build_ios_device() {
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_SYSROOT=iphoneos \
|
||||
-DGGML_METAL_TARGET_OS=ios \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
@@ -508,7 +498,6 @@ build_visionos() {
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_SYSROOT=xros \
|
||||
-DGGML_METAL_TARGET_OS=xros \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
@@ -527,7 +516,6 @@ build_visionos_sim() {
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_SYSROOT=xrsimulator \
|
||||
-DGGML_METAL_TARGET_OS=xros \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
|
||||
-DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
|
||||
-DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
|
||||
@@ -546,7 +534,6 @@ build_tvos_sim() {
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_SYSROOT=appletvsimulator \
|
||||
-DGGML_METAL_TARGET_OS=tvos \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
|
||||
-DGGML_METAL=ON \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
|
||||
@@ -565,7 +552,6 @@ build_tvos_device() {
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_SYSROOT=appletvos \
|
||||
-DGGML_METAL_TARGET_OS=tvos \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64" \
|
||||
-DGGML_METAL=ON \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \
|
||||
|
||||
+2
-2
@@ -21,13 +21,13 @@ docker run --privileged -it \
|
||||
-v $HOME/llama.cpp/ci-cache:/ci-cache \
|
||||
-v $HOME/llama.cpp/ci-results:/ci-results \
|
||||
-v $PWD:/ws -w /ws \
|
||||
registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu22.04-s5000
|
||||
mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
|
||||
```
|
||||
|
||||
Inside the container, execute the following commands:
|
||||
|
||||
```bash
|
||||
apt update -y && apt install -y bc cmake ccache git python3.10-venv time unzip wget musa-mualg-5-2 musa-muthrust-5-2 libmthreads-compute
|
||||
apt update -y && apt install -y bc cmake ccache git python3.10-venv time unzip wget
|
||||
git config --global --add safe.directory /ws
|
||||
GG_BUILD_MUSA=1 bash ./ci/run.sh /ci-results /ci-cache
|
||||
```
|
||||
|
||||
@@ -49,7 +49,17 @@ mkdir -p "$2"
|
||||
OUT=$(realpath "$1")
|
||||
MNT=$(realpath "$2")
|
||||
|
||||
# gpu-rocm self-hosted runner can't upload logs to blob; keep each run's logs in
|
||||
# their own dir keyed by the GitHub run id so an Actions run URL maps to its logs.
|
||||
if [ -n "${GG_BUILD_ROCM}" ] && [ -n "${GITHUB_RUN_ID}" ]; then
|
||||
OUT="$OUT/run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||
mkdir -p "$OUT"
|
||||
echo "ci results dir: $OUT"
|
||||
fi
|
||||
|
||||
rm -f $OUT/*.log
|
||||
rm -f $OUT/*.exit
|
||||
rm -f $OUT/*.md
|
||||
|
||||
sd=`dirname $0`
|
||||
cd $sd/../
|
||||
@@ -72,8 +82,8 @@ else
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_CUDA} ]; then
|
||||
# TODO: Drop GGML_CUDA_CCCL_VERSION when CUDA CI uses CTK >= 13.5, which bundles CCCL >= 3.5.
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CCCL_VERSION=v3.4.3"
|
||||
# TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CUB_3DOT2=ON"
|
||||
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
|
||||
@@ -150,8 +160,8 @@ if [ ! -z ${GG_BUILD_WEBGPU} ]; then
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_MUSA} ]; then
|
||||
# Use ph1 by default (MTT S5000)
|
||||
MUSA_ARCH=${MUSA_ARCH:-31}
|
||||
# Use qy1 by default (MTT S80)
|
||||
MUSA_ARCH=${MUSA_ARCH:-21}
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
|
||||
fi
|
||||
|
||||
@@ -179,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
|
||||
fi
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
|
||||
|
||||
# TODO: fix failing tests on OpenVINO backend
|
||||
CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
|
||||
# TODO: fix and re-enable the `test-llama-archs` test below
|
||||
CTEST_EXTRA="-E test-llama-archs|test-recurrent-state-rollback-nemotron-h"
|
||||
fi
|
||||
|
||||
## helpers
|
||||
@@ -201,6 +211,10 @@ function gg_wget {
|
||||
cd $cwd
|
||||
}
|
||||
|
||||
function gg_printf {
|
||||
printf -- "$@" >> $OUT/README.md
|
||||
}
|
||||
|
||||
function gg_run {
|
||||
ci=$1
|
||||
|
||||
@@ -209,10 +223,13 @@ function gg_run {
|
||||
|
||||
gg_run_$ci | tee $OUT/$ci.log
|
||||
cur=$?
|
||||
echo "$cur" > $OUT/$ci.exit
|
||||
|
||||
set +x
|
||||
set +o pipefail
|
||||
|
||||
gg_sum_$ci
|
||||
|
||||
ret=$((ret | cur))
|
||||
}
|
||||
|
||||
@@ -233,11 +250,22 @@ function gg_run_ctest_debug {
|
||||
(cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||
(time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||
|
||||
(time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
(time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_ctest_debug {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest in debug mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '\n'
|
||||
}
|
||||
|
||||
# ctest_release
|
||||
|
||||
function gg_run_ctest_release {
|
||||
@@ -262,6 +290,16 @@ function gg_run_ctest_release {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_ctest_release {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest in release mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# test_llama_archs_tensor_split
|
||||
|
||||
function gg_run_test_llama_archs_tensor_split {
|
||||
@@ -286,23 +324,14 @@ function gg_run_test_llama_archs_tensor_split {
|
||||
set +e
|
||||
}
|
||||
|
||||
# test_llama_archs_models
|
||||
function gg_sum_test_llama_archs_tensor_split {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
function gg_run_test_llama_archs_models {
|
||||
cd ${SRC}
|
||||
|
||||
set -e
|
||||
|
||||
# TODO: fix and re-enable `test-llama-archs` on OpenVINO
|
||||
# TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
|
||||
if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
|
||||
rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
|
||||
# generate the dummy models used by the model-dependent tests
|
||||
./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
|
||||
fi
|
||||
|
||||
set +e
|
||||
gg_printf 'Runs test-llama-archs with 1 to 4 devices\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# test_scripts
|
||||
@@ -318,6 +347,17 @@ function gg_run_test_scripts {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_test_scripts {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs test scripts\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-scripts.log)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '\n'
|
||||
}
|
||||
|
||||
function gg_get_model {
|
||||
#local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-f16.gguf"
|
||||
local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-q4_0.gguf"
|
||||
@@ -361,6 +401,26 @@ function gg_run_ctest_with_model_release {
|
||||
cd ..
|
||||
}
|
||||
|
||||
function gg_sum_ctest_with_model_debug {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest with model files in debug mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
function gg_sum_ctest_with_model_release {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest with model files in release mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# qwen3_0_6b
|
||||
|
||||
function gg_run_qwen3_0_6b {
|
||||
@@ -465,24 +525,50 @@ function gg_run_qwen3_0_6b {
|
||||
return 0
|
||||
}
|
||||
|
||||
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")"
|
||||
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")"
|
||||
check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
fi
|
||||
check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")"
|
||||
check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")"
|
||||
check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")"
|
||||
check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")"
|
||||
check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")"
|
||||
#check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" # note: ppl > 20.0 for this quant and model
|
||||
check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")"
|
||||
check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")"
|
||||
check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")"
|
||||
check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")"
|
||||
check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
#check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log # note: ppl > 20.0 for this quant and model
|
||||
check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||
|
||||
cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_qwen3_0_6b {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Qwen3 0.6B:\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
|
||||
gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)"
|
||||
gg_printf '- f16:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
|
||||
if [ -z ${GG_BUILD_NO_BF16} ]; then
|
||||
gg_printf '- bf16:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-bf16.log)"
|
||||
fi
|
||||
gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
|
||||
gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
|
||||
gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
|
||||
gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
|
||||
gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
|
||||
gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
|
||||
gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
|
||||
gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
|
||||
gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
|
||||
gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
|
||||
gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)"
|
||||
}
|
||||
|
||||
# bge-small
|
||||
|
||||
function gg_run_embd_bge_small {
|
||||
@@ -524,6 +610,15 @@ function gg_run_embd_bge_small {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_embd_bge_small {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'BGE Small (BERT):\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
|
||||
gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
|
||||
}
|
||||
|
||||
# rerank_tiny
|
||||
|
||||
function gg_run_rerank_tiny {
|
||||
@@ -580,102 +675,88 @@ function gg_run_rerank_tiny {
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_rerank_tiny {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Rerank Tiny (Jina):\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-rk-f16.log)"
|
||||
}
|
||||
|
||||
function gg_check_build_requirements {
|
||||
if ! command -v git &> /dev/null; then
|
||||
echo 'git not found, please install'
|
||||
gg_printf 'git not found, please install\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v git-lfs &> /dev/null; then
|
||||
echo 'git-lfs not found, please install'
|
||||
gg_printf 'git-lfs not found, please install\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! git config --get filter.lfs.clean &> /dev/null; then
|
||||
echo 'git-lfs not initialized, please run `git lfs install`'
|
||||
gg_printf 'git-lfs not initialized, please run `git lfs install`\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v wget &> /dev/null; then
|
||||
echo 'wget not found, please install'
|
||||
gg_printf 'wget not found, please install\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v python3 &> /dev/null; then
|
||||
echo 'python3 not found, please install'
|
||||
gg_printf 'python3 not found, please install\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v pip3 &> /dev/null; then
|
||||
echo 'pip3 not found, please install'
|
||||
gg_printf 'pip3 not found, please install\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! python3 -m ensurepip --help &> /dev/null; then
|
||||
echo 'ensurepip not found, please install python3-venv package'
|
||||
gg_printf 'ensurepip not found, please install python3-venv package\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v cmake &> /dev/null; then
|
||||
echo 'cmake not found, please install'
|
||||
gg_printf 'cmake not found, please install\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v ccache &> /dev/null; then
|
||||
echo 'ccache not found, please consider installing for faster builds'
|
||||
gg_printf 'ccache not found, please consider installing for faster builds\n'
|
||||
fi
|
||||
|
||||
if ! command -v ctest &> /dev/null; then
|
||||
echo 'ctest not found, please install'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v unzip &> /dev/null; then
|
||||
echo 'unzip not found, please install'
|
||||
gg_printf 'ctest not found, please install\n'
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function gg_run_test_backend_ops {
|
||||
function gg_run_test_backend_ops_cpu {
|
||||
cd ${SRC}
|
||||
|
||||
cd build-ci-release
|
||||
|
||||
set -e
|
||||
|
||||
local n_jobs=$(nproc)
|
||||
if [ "${n_jobs}" -gt 2 ]; then
|
||||
n_jobs=2
|
||||
fi
|
||||
local args_extra="-j ${n_jobs}"
|
||||
|
||||
# TODO: fix multi-threaded for ROCm
|
||||
# https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
|
||||
if [ ! -z ${GG_BUILD_ROCM} ]; then
|
||||
args_extra=""
|
||||
fi
|
||||
|
||||
# TODO: MoltenVK bug?
|
||||
# https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
|
||||
if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
|
||||
args_extra=""
|
||||
fi
|
||||
|
||||
# TODO: OpenVINO GPU plugin crashes (CL_OUT_OF_RESOURCES) with 2 concurrent workers on GPU.
|
||||
if [ ! -z "${GG_BUILD_OPENVINO}" ] && [ "${GGML_OPENVINO_DEVICE:-}" = "GPU" ]; then
|
||||
args_extra=""
|
||||
fi
|
||||
|
||||
# TODO: reduce the test-backend-ops timeout to 1800s
|
||||
if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
|
||||
(time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
||||
else
|
||||
(time timeout 3600 ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
|
||||
fi
|
||||
(time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_test_backend_ops_cpu {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs test-backend-ops for CPU backend\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '\n'
|
||||
}
|
||||
|
||||
## main
|
||||
|
||||
export LLAMA_ARG_LOG_PREFIX=1
|
||||
@@ -704,11 +785,12 @@ ret=0
|
||||
test $ret -eq 0 && gg_run ctest_debug
|
||||
test $ret -eq 0 && gg_run ctest_release
|
||||
|
||||
test $ret -eq 0 && gg_run test_backend_ops
|
||||
|
||||
test $ret -eq 0 && gg_run test_llama_archs_models
|
||||
test $ret -eq 0 && gg_run test_llama_archs_tensor_split
|
||||
|
||||
if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
|
||||
test $ret -eq 0 && gg_run test_backend_ops_cpu
|
||||
fi
|
||||
|
||||
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||
test $ret -eq 0 && gg_run embd_bge_small
|
||||
test $ret -eq 0 && gg_run rerank_tiny
|
||||
@@ -723,4 +805,6 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||
test $ret -eq 0 && gg_run ctest_with_model_release
|
||||
fi
|
||||
|
||||
cat $OUT/README.md
|
||||
|
||||
exit $ret
|
||||
|
||||
@@ -17,16 +17,14 @@ find_library(llama_LIBRARY llama
|
||||
NO_CMAKE_FIND_ROOT_PATH
|
||||
)
|
||||
|
||||
if(NOT TARGET llama)
|
||||
add_library(llama UNKNOWN IMPORTED)
|
||||
set_target_properties(llama
|
||||
PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
|
||||
IMPORTED_LOCATION "${llama_LIBRARY}"
|
||||
INTERFACE_COMPILE_FEATURES c_std_90
|
||||
POSITION_INDEPENDENT_CODE ON)
|
||||
endif()
|
||||
add_library(llama UNKNOWN IMPORTED)
|
||||
set_target_properties(llama
|
||||
PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
|
||||
IMPORTED_LOCATION "${llama_LIBRARY}"
|
||||
INTERFACE_COMPILE_FEATURES c_std_90
|
||||
POSITION_INDEPENDENT_CODE ON)
|
||||
|
||||
check_required_components(Llama)
|
||||
|
||||
@@ -53,10 +53,7 @@ endif()
|
||||
|
||||
set(TARGET llama-common)
|
||||
|
||||
include(parsers/sources.cmake)
|
||||
|
||||
add_library(${TARGET}
|
||||
${LLAMA_CHAT_PARSERS_SOURCES}
|
||||
arg.cpp
|
||||
arg.h
|
||||
base64.hpp
|
||||
@@ -84,8 +81,6 @@ add_library(${TARGET}
|
||||
imatrix-loader.cpp
|
||||
imatrix-loader.h
|
||||
json-schema-to-grammar.cpp
|
||||
json-schema.cpp
|
||||
json-schema.h
|
||||
json.cpp
|
||||
json.h
|
||||
llguidance.cpp
|
||||
|
||||
+72
-97
@@ -351,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
|
||||
static std::string get_default_local_path(const std::string & url) {
|
||||
auto f = string_split<std::string>(url, '#').front();
|
||||
f = string_split<std::string>(f, '?').front();
|
||||
return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
|
||||
return fs_get_cache_file(string_split<std::string>(f, '/').back());
|
||||
}
|
||||
|
||||
static bool spec_types_is_default(const common_params & params) {
|
||||
@@ -387,9 +387,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (curr_ex == LLAMA_EXAMPLE_DOWNLOAD) {
|
||||
use_mmproj = true;
|
||||
}
|
||||
|
||||
opts.bearer_token = params.hf_token;
|
||||
opts.offline = params.offline;
|
||||
@@ -682,10 +679,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
// if HF repo is a preset repo, we simply run server in router mode with the preset.ini file
|
||||
params.models_preset_hf = params.model.hf_repo; // only for showing a warning
|
||||
params.models_preset = hf_cache::finalize_file(plan.preset);
|
||||
// clear the model so the server starts in router mode
|
||||
params.model.path.clear();
|
||||
params.model.hf_repo.clear();
|
||||
params.model.docker_repo.clear();
|
||||
params.model = common_params_model{}; // make sure to clear model, so server starts in router mode
|
||||
});
|
||||
}
|
||||
|
||||
@@ -723,24 +717,24 @@ void common_models_handler_apply(common_models_handler & handler, common_params
|
||||
// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)
|
||||
// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)
|
||||
static void common_params_apply_system_config(common_params & params, llama_example ex) {
|
||||
std::vector<std::filesystem::path> paths;
|
||||
std::vector<std::string> paths;
|
||||
|
||||
#if defined(_WIN32)
|
||||
const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA");
|
||||
const std::string program_data = common_get_env("PROGRAMDATA");
|
||||
if (!program_data.empty()) {
|
||||
paths.push_back(program_data / "llama.cpp" / "config.ini");
|
||||
paths.push_back(program_data + "\\llama.cpp\\config.ini");
|
||||
}
|
||||
#else
|
||||
paths.push_back("/etc/llama.cpp/config.ini");
|
||||
#endif
|
||||
|
||||
try {
|
||||
paths.push_back(fs_get_config_directory() / "config.ini");
|
||||
paths.push_back(fs_get_config_directory() + "config.ini");
|
||||
} catch (const std::exception & e) {
|
||||
LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());
|
||||
}
|
||||
|
||||
std::vector<std::filesystem::path> found;
|
||||
std::vector<std::string> found;
|
||||
for (const auto & path : paths) {
|
||||
std::error_code ec;
|
||||
if (std::filesystem::exists(path, ec)) {
|
||||
@@ -754,7 +748,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
|
||||
common_preset_context ctx(ex);
|
||||
ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
|
||||
for (const auto & path : found) {
|
||||
LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
|
||||
LOG_INF("using config file: %s\n", path.c_str());
|
||||
common_preset global;
|
||||
common_presets presets = ctx.load_from_ini(path, global);
|
||||
global.apply_to_params(params);
|
||||
@@ -878,6 +872,17 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
arg.c_str(), e.what(), opt.to_string().c_str()));
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: remove this check after deprecating --mmap|mlock|dio
|
||||
auto has_arg = [&](std::initializer_list<const char *> names) {
|
||||
return std::any_of(names.begin(), names.end(), [&](const char * name) {
|
||||
return seen_args.count(name);
|
||||
});
|
||||
};
|
||||
if (has_arg({"-lm", "--load-mode"}) &&
|
||||
has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) {
|
||||
LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n");
|
||||
}
|
||||
};
|
||||
|
||||
// parse all CLI args now, so that -hf is available below for remote preset resolution
|
||||
@@ -889,12 +894,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
postprocess_cpu_params(params.speculative.draft.cpuparams, ¶ms.cpuparams);
|
||||
postprocess_cpu_params(params.speculative.draft.cpuparams_batch, ¶ms.cpuparams_batch);
|
||||
|
||||
// default the mmproj device to the global device selection if not set explicitly with -mmdev
|
||||
if (params.mmproj_use_gpu && params.mmproj_device == nullptr && !params.devices.empty()) {
|
||||
params.mmproj_device = params.devices.front();
|
||||
params.mmproj_use_gpu = params.mmproj_device != nullptr;
|
||||
}
|
||||
|
||||
if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {
|
||||
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
|
||||
}
|
||||
@@ -961,11 +960,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
));
|
||||
}
|
||||
|
||||
// if the preserve_reasoning kwarg was not specified explicitly, enable it by default
|
||||
if (!params.default_template_kwargs.count("preserve_reasoning")) {
|
||||
params.default_template_kwargs["preserve_reasoning"] = "true";
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -2022,7 +2016,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.sampling.temp = std::max(params.sampling.temp, 0.0f);
|
||||
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP;
|
||||
}
|
||||
).set_sampling().set_env("LLAMA_ARG_TEMPERATURE"));
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--top-k"}, "N",
|
||||
string_format("top-k sampling (default: %d, 0 = disabled)", params.sampling.top_k),
|
||||
@@ -2038,7 +2032,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.sampling.top_p = std::stof(value);
|
||||
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_P;
|
||||
}
|
||||
).set_sampling().set_env("LLAMA_ARG_TOP_P"));
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--min-p"}, "N",
|
||||
string_format("min-p sampling (default: %.2f, 0.0 = disabled)", (double)params.sampling.min_p),
|
||||
@@ -2046,7 +2040,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.sampling.min_p = std::stof(value);
|
||||
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIN_P;
|
||||
}
|
||||
).set_sampling().set_env("LLAMA_ARG_MIN_P"));
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--top-nsigma", "--top-n-sigma"}, "N",
|
||||
string_format("top-n-sigma sampling (default: %.2f, -1.0 = disabled)", params.sampling.top_n_sigma),
|
||||
@@ -2102,7 +2096,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.sampling.penalty_repeat = penalty_repeat;
|
||||
params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT;
|
||||
}
|
||||
).set_sampling().set_env("LLAMA_ARG_REPEAT_PENALTY"));
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--presence-penalty"}, "N",
|
||||
string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present),
|
||||
@@ -2113,7 +2107,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
}
|
||||
params.sampling.penalty_present = penalty_present;
|
||||
}
|
||||
).set_sampling().set_env("LLAMA_ARG_PRESENCE_PENALTY"));
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--frequency-penalty"}, "N",
|
||||
string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq),
|
||||
@@ -2124,7 +2118,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
}
|
||||
params.sampling.penalty_freq = penalty_freq;
|
||||
}
|
||||
).set_sampling().set_env("LLAMA_ARG_FREQUENCY_PENALTY"));
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"--dry-multiplier"}, "N",
|
||||
string_format("set DRY sampling multiplier (default: %.2f, 0.0 = disabled)", (double)params.sampling.dry_multiplier),
|
||||
@@ -2283,14 +2277,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"-j", "--json-schema"}, "SCHEMA",
|
||||
"JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
|
||||
"JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, json_schema_to_grammar(json::parse(value))};
|
||||
}
|
||||
).set_sampling());
|
||||
add_opt(common_arg(
|
||||
{"-jf", "--json-schema-file"}, "FILE",
|
||||
"File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
|
||||
"File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
|
||||
[](common_params & params, const std::string & value) {
|
||||
std::ifstream file(value);
|
||||
if (!file) {
|
||||
@@ -2611,7 +2605,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
// note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
|
||||
{"-mmdev", "--mmproj-device"}, "DEVICE",
|
||||
"device to use for multimodal projector (none = don't offload, default: follows --device)\n"
|
||||
"device to use for multimodal projector (none = don't offload, default: auto)\n"
|
||||
"use --list-devices to see a list of available devices",
|
||||
[](common_params & params, const std::string & value) {
|
||||
if (value == "none") {
|
||||
@@ -2679,17 +2673,42 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.video_ffmpeg_bin_dir = value;
|
||||
}
|
||||
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));
|
||||
add_opt(common_arg(
|
||||
{"--rpc"}, "SERVERS",
|
||||
"comma-separated list of RPC servers (host:port)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
if (!llama_supports_rpc()) {
|
||||
throw std::invalid_argument("RPC not supported in this build");
|
||||
if (params.is_gen_docs || llama_supports_rpc()) {
|
||||
add_opt(common_arg(
|
||||
{"--rpc"}, "SERVERS",
|
||||
"comma-separated list of RPC servers (host:port)",
|
||||
[](common_params & params, const std::string & value) {
|
||||
add_rpc_devices(value);
|
||||
GGML_UNUSED(params);
|
||||
}
|
||||
add_rpc_devices(value);
|
||||
GGML_UNUSED(params);
|
||||
).set_env("LLAMA_ARG_RPC"));
|
||||
}
|
||||
add_opt(common_arg(
|
||||
{"--mlock"},
|
||||
"DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing",
|
||||
[](common_params & params) {
|
||||
LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");
|
||||
params.load_mode = LLAMA_LOAD_MODE_MLOCK;
|
||||
}
|
||||
).set_env("LLAMA_ARG_RPC"));
|
||||
).set_env("LLAMA_ARG_MLOCK"));
|
||||
add_opt(common_arg(
|
||||
{"--mmap"},
|
||||
{"--no-mmap"},
|
||||
"DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)",
|
||||
[](common_params & params, bool value) {
|
||||
LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n");
|
||||
params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE;
|
||||
}
|
||||
).set_env("LLAMA_ARG_MMAP"));
|
||||
add_opt(common_arg(
|
||||
{"-dio", "--direct-io"},
|
||||
{"-ndio", "--no-direct-io"},
|
||||
"DEPRECATED in favor of `--load-mode`: use DirectIO if available",
|
||||
[](common_params & params, bool value) {
|
||||
LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n");
|
||||
params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE;
|
||||
}
|
||||
).set_env("LLAMA_ARG_DIO"));
|
||||
add_opt(common_arg(
|
||||
{"-lm", "--load-mode"}, "MODE",
|
||||
"model loading mode (default: auto)\n"
|
||||
@@ -2710,18 +2729,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
}
|
||||
).set_env("LLAMA_ARG_LOAD_MODE"));
|
||||
add_opt(common_arg(
|
||||
{"-lzm", "--lazy-mode"}, "MODE",
|
||||
{"--tensor-read-lazy"}, "MODE",
|
||||
"on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
|
||||
"- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
|
||||
"- auto: on, but only for tensors larger than 4 GiB\n"
|
||||
"- off: always keep them resident",
|
||||
[](common_params & params, const std::string & value) {
|
||||
/**/ if (value == "on") { params.lazy_mode = LLAMA_LAZY_MODE_ON; }
|
||||
else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
|
||||
else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; }
|
||||
/**/ if (value == "on") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON; }
|
||||
else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
|
||||
else if (value == "off") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF; }
|
||||
else { throw std::invalid_argument("invalid value"); }
|
||||
}
|
||||
).set_env("LLAMA_ARG_LAZY_MODE"));
|
||||
).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));
|
||||
add_opt(common_arg(
|
||||
{"--numa"}, "TYPE",
|
||||
"attempt optimizations that help on some NUMA systems\n"
|
||||
@@ -3183,13 +3202,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.process_output = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
|
||||
add_opt(common_arg(
|
||||
{"--nextn"},
|
||||
string_format("collect data for MTP/NextN layers (default: %s)", params.load_mtp ? "true" : "false"),
|
||||
[](common_params & params) {
|
||||
params.load_mtp = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
|
||||
add_opt(common_arg(
|
||||
{"--ppl"},
|
||||
{"--no-ppl"},
|
||||
@@ -3322,18 +3334,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
).set_examples({LLAMA_EXAMPLE_EMBEDDING}));
|
||||
add_opt(common_arg(
|
||||
{"--host"}, "HOST",
|
||||
string_format("IP addresses to listen on, comma-separated, or UNIX socket paths ending in .sock; with multiple TCP addresses, :: binds IPv6 only; overlapping addresses result in undefined behavior (default: %s)", params.hostnames[0].c_str()),
|
||||
string_format("ip address to listen, or bind to an UNIX socket if the address ends with .sock (default: %s)", params.hostname.c_str()),
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.hostnames.clear();
|
||||
for (auto & host : parse_csv_row(value)) {
|
||||
host = string_strip(host);
|
||||
if (!host.empty()) {
|
||||
params.hostnames.push_back(host);
|
||||
}
|
||||
}
|
||||
if (params.hostnames.empty()) {
|
||||
throw std::invalid_argument("--host requires at least one address");
|
||||
}
|
||||
params.hostname = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HOST"));
|
||||
add_opt(common_arg(
|
||||
@@ -3550,10 +3553,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
LOG_WRN("Setting 'enable_thinking' via --chat-template-kwargs is deprecated. "
|
||||
"Use --reasoning on / --reasoning off instead.\n");
|
||||
}
|
||||
if (item.key() == "preserve_reasoning") {
|
||||
LOG_WRN("Setting 'preserve_reasoning' via --chat-template-kwargs is deprecated. "
|
||||
"Use --reasoning-preserve / --no-reasoning-preserve instead.\n");
|
||||
}
|
||||
params.default_template_kwargs[item.key()] = item.value().dump();
|
||||
}
|
||||
}
|
||||
@@ -3744,7 +3743,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--reasoning-preserve"},
|
||||
{"--no-reasoning-preserve"},
|
||||
"preserve reasoning trace in the full history, not just the last assistant message (default: enabled)\n"
|
||||
"preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n"
|
||||
"compatible with certain templates having 'supports_preserve_reasoning' capability\n"
|
||||
"example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking",
|
||||
[](common_params & params, bool value) {
|
||||
@@ -3753,7 +3752,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
} else {
|
||||
params.default_template_kwargs["preserve_reasoning"] = "false";
|
||||
}
|
||||
params.preserve_reasoning_specified = true;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE"));
|
||||
add_opt(common_arg(
|
||||
@@ -3893,14 +3891,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
common_log_set_file(common_log_main(), value.c_str());
|
||||
}
|
||||
).set_env("LLAMA_ARG_LOG_FILE"));
|
||||
add_opt(common_arg(
|
||||
{"--log-jsonl"},
|
||||
{"--no-log-jsonl"},
|
||||
"Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)",
|
||||
[](common_params &, bool value) {
|
||||
common_log_set_jsonl(value);
|
||||
}
|
||||
).set_env("LLAMA_ARG_LOG_JSONL"));
|
||||
add_opt(common_arg(
|
||||
{"--log-prompts-dir"}, "PATH",
|
||||
"Log prompts to directory (auto-created if not present; only used for debugging, default: disabled)",
|
||||
@@ -4219,24 +4209,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.speculative.draft.backend_sampling = value;
|
||||
}
|
||||
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING"));
|
||||
add_opt(common_arg(
|
||||
{"--spec-draft-sampling"}, "{greedy,probabilistic}",
|
||||
string_format("how the draft is sampled: greedy takes its argmax, probabilistic samples it and has "
|
||||
"the target verify by rejection sampling (default: %s)",
|
||||
params.speculative.draft.probabilistic ? "probabilistic" : "greedy"),
|
||||
[](common_params & params, const std::string & value) {
|
||||
if (value == "greedy") {
|
||||
params.speculative.draft.probabilistic = false;
|
||||
} else if (value == "probabilistic") {
|
||||
params.speculative.draft.probabilistic = true;
|
||||
} else {
|
||||
throw std::invalid_argument("invalid value, must be one of: greedy, probabilistic");
|
||||
}
|
||||
}
|
||||
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_SAMPLING"));
|
||||
add_opt(common_arg(
|
||||
{"--spec-draft-device", "-devd", "--device-draft"}, "<dev1,dev2,..>",
|
||||
"comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)\n"
|
||||
"comma-separated list of devices to use for offloading the draft model (none = don't offload)\n"
|
||||
"use --list-devices to see a list of available devices",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.speculative.draft.devices = parse_device_list(value);
|
||||
@@ -4269,7 +4244,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.speculative.draft.mparams.path = value;
|
||||
params.speculative.draft.mparams.hf_file = value; // will be used if --spec-draft-hf is set
|
||||
}
|
||||
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_IMATRIX}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL"));
|
||||
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL"));
|
||||
add_opt(common_arg(
|
||||
{"--spec-type"}, common_speculative_all_types_str(),
|
||||
string_format("comma-separated list of types of speculative decoding to use (default: %s)\n",
|
||||
|
||||
@@ -122,8 +122,6 @@ struct common_params_context {
|
||||
|
||||
// parse input arguments from CLI
|
||||
// if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message)
|
||||
// TODO: this function can load ggml backend (by calling llama_support_rpc)
|
||||
// this is a side-effect that should be avoided
|
||||
bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);
|
||||
|
||||
// load all backends and print the list of available (non-CPU) devices to stdout
|
||||
|
||||
@@ -29,7 +29,7 @@ const char * llama_build_info(void) {
|
||||
return s.c_str();
|
||||
}
|
||||
|
||||
void llama_print_build_info(const char * llama_version, FILE * stream) {
|
||||
fprintf(stream, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
|
||||
fprintf(stream, "built with %s for %s\n", llama_compiler(), llama_build_target());
|
||||
void llama_print_build_info(const char * llama_version) {
|
||||
fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
|
||||
fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
|
||||
}
|
||||
|
||||
+1
-3
@@ -1,7 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
int llama_build_number(void);
|
||||
|
||||
const char * llama_commit(void);
|
||||
@@ -10,4 +8,4 @@ const char * llama_compiler(void);
|
||||
const char * llama_build_target(void);
|
||||
const char * llama_build_info(void);
|
||||
|
||||
void llama_print_build_info(const char *, FILE * = stderr);
|
||||
void llama_print_build_info(const char *);
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
#include "common.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "log.h"
|
||||
#include "parsers/parsers.h"
|
||||
#include "peg-parser.h"
|
||||
|
||||
#include <stdexcept>
|
||||
@@ -13,6 +12,16 @@
|
||||
|
||||
using json = common_json;
|
||||
|
||||
// Helper to iterate over tools/functions
|
||||
static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
|
||||
for (const auto & tool : tools) {
|
||||
if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
|
||||
continue;
|
||||
}
|
||||
fn(tool);
|
||||
}
|
||||
}
|
||||
|
||||
namespace autoparser {
|
||||
|
||||
parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :
|
||||
@@ -78,6 +87,15 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
auto schema = function.contains("parameters") ? function.at("parameters") : json::object();
|
||||
builder.resolve_refs(schema);
|
||||
});
|
||||
if (has_response_format) {
|
||||
auto schema = inputs.json_schema;
|
||||
builder.resolve_refs(schema);
|
||||
}
|
||||
parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
@@ -294,7 +312,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & func = tool.at("function");
|
||||
std::string name = func.at("name");
|
||||
const auto schema = common_chat_tool_parameters(func);
|
||||
const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object();
|
||||
|
||||
// Build call_id parser based on position (if supported)
|
||||
bool have_call_id = false;
|
||||
@@ -365,31 +383,43 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
|
||||
common_peg_parser tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](const json & tool) {
|
||||
const auto & func = tool.at("function");
|
||||
std::string name = func.at("name");
|
||||
const auto & func = tool.at("function");
|
||||
std::string name = func.at("name");
|
||||
auto params = func.contains("parameters") ? func.at("parameters") : json::object();
|
||||
const auto & properties = params.contains("properties") ? params.at("properties") : json::object();
|
||||
|
||||
std::set<std::string> required;
|
||||
if (params.contains("required")) {
|
||||
required = params.at("required").get<std::set<std::string>>();
|
||||
}
|
||||
|
||||
auto schema_info = common_schema_info();
|
||||
schema_info.resolve_refs(params);
|
||||
|
||||
// Build parser for each argument, separating required and optional
|
||||
std::vector<common_peg_parser> required_parsers;
|
||||
std::vector<common_peg_parser> optional_parsers;
|
||||
foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
for (const auto & [param_name, param_schema] : properties.items()) {
|
||||
bool is_required = required.find(param_name) != required.end();
|
||||
|
||||
auto arg =
|
||||
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
|
||||
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) +
|
||||
arguments.name_suffix) +
|
||||
arguments.value_prefix +
|
||||
(param.schema->may_be_string() ?
|
||||
(schema_info.resolves_to_string(param_schema) ?
|
||||
p.ac(p.tool_arg_string_value(until_suffix) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
|
||||
(p.tool_arg_json_value(p.schema(
|
||||
p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) +
|
||||
p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +
|
||||
p.tool_arg_close(p.literal(arguments.value_suffix)))));
|
||||
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
|
||||
if (param.required) {
|
||||
auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
|
||||
if (is_required) {
|
||||
required_parsers.push_back(named_arg);
|
||||
} else {
|
||||
optional_parsers.push_back(named_arg);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Build required arg sequence in definition order
|
||||
common_peg_parser args_seq = p.eps();
|
||||
|
||||
+11
-12
@@ -318,13 +318,13 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
|
||||
bool is_content = node.tag == common_chat_peg_builder::CONTENT;
|
||||
|
||||
if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here
|
||||
result.reasoning_content += node.sanitized_text();
|
||||
result.reasoning_content += std::string(node.text);
|
||||
}
|
||||
|
||||
if (is_content) {
|
||||
// Concatenate content from multiple content nodes (e.g., when reasoning markers
|
||||
// are preserved before content markers in reasoning_format=NONE mode)
|
||||
result.content += node.sanitized_text();
|
||||
result.content += std::string(node.text);
|
||||
}
|
||||
|
||||
// Handle tool-related tags (supporting both JSON and tagged formats)
|
||||
@@ -451,7 +451,6 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
|
||||
result.tool_calls.push_back(pending_tool_call.value());
|
||||
}
|
||||
pending_tool_call.reset();
|
||||
current_tool = nullptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -489,7 +488,7 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
|
||||
// Build argument parsers
|
||||
auto args = eps();
|
||||
@@ -566,7 +565,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
|
||||
auto args = eps();
|
||||
if (params.contains("properties") && !params["properties"].empty()) {
|
||||
@@ -641,7 +640,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
|
||||
// Build inner object fields
|
||||
std::vector<common_peg_parser> inner_fields;
|
||||
@@ -727,7 +726,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
|
||||
auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +
|
||||
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
|
||||
@@ -796,7 +795,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
|
||||
}
|
||||
const auto & function = tool_def.at("function");
|
||||
std::string name = function.at("name");
|
||||
ordered_json params = common_chat_tool_parameters(function);
|
||||
ordered_json params = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
|
||||
|
||||
auto tool_name_ = name_key_parser + space() + literal(":") + space() +
|
||||
atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
|
||||
@@ -1059,12 +1058,12 @@ void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, co
|
||||
const auto & node = arena.get(id);
|
||||
|
||||
if (node.tag == "reasoning") {
|
||||
result.reasoning_content += node.sanitized_text();
|
||||
result.reasoning_content += std::string(node.text);
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.tag == "content") {
|
||||
result.content += node.sanitized_text();
|
||||
result.content += std::string(node.text);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1207,12 +1206,12 @@ void common_chat_peg_minimax_m3_mapper::visit(const common_peg_ast_arena & arena
|
||||
const auto & node = arena.get(id);
|
||||
|
||||
if (node.tag == common_chat_peg_builder::REASONING) {
|
||||
result.reasoning_content += node.sanitized_text();
|
||||
result.reasoning_content += std::string(node.text);
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.tag == common_chat_peg_builder::CONTENT) {
|
||||
result.content += node.sanitized_text();
|
||||
result.content += std::string(node.text);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
+2412
-36
File diff suppressed because it is too large
Load Diff
@@ -360,9 +360,6 @@ common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg
|
||||
|
||||
common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
|
||||
|
||||
// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments.
|
||||
common_json common_chat_tool_parameters(const common_json & function);
|
||||
|
||||
// get template caps, useful for reporting to server /props endpoint
|
||||
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
|
||||
|
||||
|
||||
+296
-362
@@ -1,12 +1,8 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpp.h"
|
||||
#include "gguf.h"
|
||||
|
||||
#include "build-info.h"
|
||||
#include "common.h"
|
||||
|
||||
#include "../src/llama-ext.h"
|
||||
|
||||
#include "fit.h"
|
||||
#include "log.h"
|
||||
#include "llama.h"
|
||||
@@ -50,10 +46,11 @@
|
||||
#include <io.h>
|
||||
#else
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#if !defined(_WIN32)
|
||||
#if defined(__linux__)
|
||||
#include <sys/types.h>
|
||||
#include <pwd.h>
|
||||
#endif
|
||||
@@ -617,6 +614,34 @@ std::string string_from(const struct llama_context * ctx, const std::vector<llam
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch) {
|
||||
std::stringstream buf;
|
||||
|
||||
buf << "[ ";
|
||||
|
||||
bool first = true;
|
||||
for (int i = 0; i < batch.n_tokens; ++i) {
|
||||
if (!first) {
|
||||
buf << ", ";
|
||||
} else {
|
||||
first = false;
|
||||
}
|
||||
|
||||
auto detokenized = common_token_to_piece(ctx, batch.token[i]);
|
||||
|
||||
buf << "\n" << std::to_string(i)
|
||||
<< ", token '" << detokenized << "'"
|
||||
<< ", pos " << std::to_string(batch.pos[i])
|
||||
<< ", n_seq_id " << std::to_string(batch.n_seq_id[i])
|
||||
<< ", seq_id " << std::to_string(batch.seq_id[i][0])
|
||||
<< ", logits " << std::to_string(batch.logits[i]);
|
||||
}
|
||||
|
||||
buf << " ]";
|
||||
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
void string_process_escapes(std::string & input) {
|
||||
std::size_t input_len = input.length();
|
||||
std::size_t output_idx = 0;
|
||||
@@ -875,7 +900,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
|
||||
|
||||
|
||||
#ifdef _WIN32
|
||||
std::wstring utf8_to_wstring(const std::string & str) {
|
||||
static std::wstring utf8_to_wstring(const std::string & str) {
|
||||
if (str.empty()) {
|
||||
return std::wstring();
|
||||
}
|
||||
@@ -891,52 +916,82 @@ std::wstring utf8_to_wstring(const std::string & str) {
|
||||
|
||||
return wstr;
|
||||
}
|
||||
|
||||
std::string wstring_to_utf8(const std::wstring & str) {
|
||||
if (str.empty()) {
|
||||
return std::string();
|
||||
}
|
||||
|
||||
int size = WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0, NULL, NULL);
|
||||
|
||||
if (size <= 0) {
|
||||
return std::string();
|
||||
}
|
||||
|
||||
std::string utf8(size, 0);
|
||||
WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), &utf8[0], size, NULL, NULL);
|
||||
|
||||
return utf8;
|
||||
}
|
||||
#endif
|
||||
|
||||
// returns the path as a UTF-8 string, preserving its separators
|
||||
std::string fs_path_to_utf8(const std::filesystem::path & path) {
|
||||
const auto value = path.u8string();
|
||||
return std::string(value.begin(), value.end());
|
||||
}
|
||||
// returns true if successful, false otherwise
|
||||
bool fs_create_directory_with_parents(const std::string & path) {
|
||||
#ifdef _WIN32
|
||||
std::wstring wpath = utf8_to_wstring(path);
|
||||
|
||||
void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
|
||||
std::error_code ec;
|
||||
std::filesystem::path path_tmp = path;
|
||||
path_tmp += ".tmp";
|
||||
|
||||
if (path.has_parent_path()) {
|
||||
std::filesystem::create_directories(path.parent_path(), ec);
|
||||
// if the path already exists, check whether it's a directory
|
||||
const DWORD attributes = GetFileAttributesW(wpath.c_str());
|
||||
if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
std::ofstream file(path_tmp, std::ios::binary);
|
||||
file << data;
|
||||
file.close();
|
||||
size_t pos_slash = 0;
|
||||
|
||||
if (!file.fail()) {
|
||||
std::filesystem::rename(path_tmp, path, ec);
|
||||
// process path from front to back, procedurally creating directories
|
||||
while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {
|
||||
const std::wstring subpath = wpath.substr(0, pos_slash);
|
||||
|
||||
pos_slash += 1;
|
||||
|
||||
// skip the drive letter, in some systems it can return an access denied error
|
||||
if (subpath.length() == 2 && subpath[1] == ':') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const bool success = CreateDirectoryW(subpath.c_str(), NULL);
|
||||
|
||||
if (!success) {
|
||||
const DWORD error = GetLastError();
|
||||
|
||||
// if the path already exists, ensure that it's a directory
|
||||
if (error == ERROR_ALREADY_EXISTS) {
|
||||
const DWORD attributes = GetFileAttributesW(subpath.c_str());
|
||||
if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (file.fail() || ec) {
|
||||
std::filesystem::remove(path_tmp, ec);
|
||||
throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
|
||||
return true;
|
||||
#else
|
||||
// if the path already exists, check whether it's a directory
|
||||
struct stat info;
|
||||
if (stat(path.c_str(), &info) == 0) {
|
||||
return S_ISDIR(info.st_mode);
|
||||
}
|
||||
|
||||
size_t pos_slash = 1; // skip leading slashes for directory creation
|
||||
|
||||
// process path from front to back, procedurally creating directories
|
||||
while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {
|
||||
const std::string subpath = path.substr(0, pos_slash);
|
||||
struct stat info;
|
||||
|
||||
// if the path already exists, ensure that it's a directory
|
||||
if (stat(subpath.c_str(), &info) == 0) {
|
||||
if (!S_ISDIR(info.st_mode)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// create parent directories
|
||||
const int ret = mkdir(subpath.c_str(), 0755);
|
||||
if (ret != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
pos_slash += 1;
|
||||
}
|
||||
|
||||
return true;
|
||||
#endif // _WIN32
|
||||
}
|
||||
|
||||
bool fs_is_directory(const std::string & path) {
|
||||
@@ -961,91 +1016,172 @@ void common_set_env(const std::string & name, const std::string & value) {
|
||||
#endif
|
||||
}
|
||||
|
||||
std::filesystem::path common_get_path_from_env(const std::string & name) {
|
||||
#if defined(_WIN32)
|
||||
const std::wstring wname = utf8_to_wstring(name);
|
||||
const wchar_t * wvalue = _wgetenv(wname.c_str());
|
||||
return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
|
||||
#else
|
||||
const char * value = std::getenv(name.c_str());
|
||||
return value ? std::filesystem::path(value) : std::filesystem::path();
|
||||
#endif
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
static std::filesystem::path get_home_directory() {
|
||||
std::filesystem::path home = common_get_path_from_env("HOME");
|
||||
if (!home.empty()) {
|
||||
return home;
|
||||
}
|
||||
const struct passwd * pw = getpwuid(getuid());
|
||||
if (!pw || !pw->pw_dir || !*pw->pw_dir) {
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
}
|
||||
return pw->pw_dir;
|
||||
}
|
||||
#endif
|
||||
|
||||
std::filesystem::path fs_get_cache_directory() {
|
||||
std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");
|
||||
if (!cache_directory.empty()) {
|
||||
return cache_directory;
|
||||
}
|
||||
#if defined(_WIN32)
|
||||
cache_directory = common_get_path_from_env("LOCALAPPDATA");
|
||||
std::string fs_get_cache_directory() {
|
||||
std::string cache_directory = "";
|
||||
auto ensure_trailing_slash = [](std::string p) {
|
||||
// Make sure to add trailing slash
|
||||
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
|
||||
p += DIRECTORY_SEPARATOR;
|
||||
}
|
||||
return p;
|
||||
};
|
||||
cache_directory = common_get_env("LLAMA_CACHE");
|
||||
if (cache_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
|
||||
}
|
||||
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
|
||||
defined(__OpenBSD__) || defined(__NetBSD__)
|
||||
const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");
|
||||
const std::string home = common_get_env("HOME");
|
||||
if (!xdg_cache_home.empty()) {
|
||||
cache_directory = xdg_cache_home;
|
||||
} else if (!home.empty()) {
|
||||
cache_directory = home + "/.cache/";
|
||||
} else {
|
||||
#if defined(__linux__)
|
||||
/* no $HOME is defined, fallback to getpwuid */
|
||||
struct passwd *pw = getpwuid(getuid());
|
||||
if ((!pw) || (!pw->pw_dir)) {
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
}
|
||||
|
||||
cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");
|
||||
#else /* defined(__linux__) */
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
#endif /* defined(__linux__) */
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
cache_directory = get_home_directory() / "Library/Caches";
|
||||
cache_directory = common_get_env("HOME");
|
||||
if (cache_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
}
|
||||
cache_directory += "/Library/Caches/";
|
||||
#elif defined(_WIN32)
|
||||
cache_directory = common_get_env("LOCALAPPDATA");
|
||||
if (cache_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
|
||||
}
|
||||
#elif defined(__EMSCRIPTEN__)
|
||||
GGML_ABORT("not implemented on this platform");
|
||||
#else
|
||||
cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
|
||||
if (cache_directory.empty()) {
|
||||
cache_directory = get_home_directory() / ".cache";
|
||||
}
|
||||
# error Unknown architecture
|
||||
#endif
|
||||
return cache_directory / "llama.cpp";
|
||||
cache_directory = ensure_trailing_slash(cache_directory);
|
||||
cache_directory += "llama.cpp";
|
||||
}
|
||||
return ensure_trailing_slash(cache_directory);
|
||||
}
|
||||
|
||||
std::filesystem::path fs_get_config_directory() {
|
||||
std::filesystem::path config_directory;
|
||||
#if defined(_WIN32)
|
||||
config_directory = common_get_path_from_env("APPDATA");
|
||||
std::string fs_get_config_directory() {
|
||||
std::string config_directory = "";
|
||||
auto ensure_trailing_slash = [](std::string p) {
|
||||
if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
|
||||
p += DIRECTORY_SEPARATOR;
|
||||
}
|
||||
return p;
|
||||
};
|
||||
#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
|
||||
defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
|
||||
const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
|
||||
const std::string home = common_get_env("HOME");
|
||||
if (!xdg_config_home.empty()) {
|
||||
config_directory = xdg_config_home;
|
||||
} else if (!home.empty()) {
|
||||
config_directory = home + "/.config/";
|
||||
} else {
|
||||
#if defined(__linux__)
|
||||
/* no $HOME is defined, fallback to getpwuid */
|
||||
struct passwd *pw = getpwuid(getuid());
|
||||
if ((!pw) || (!pw->pw_dir)) {
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
}
|
||||
|
||||
config_directory = std::string(pw->pw_dir) + std::string("/.config/");
|
||||
#else
|
||||
throw std::runtime_error("Failed to find $HOME directory");
|
||||
#endif
|
||||
}
|
||||
#elif defined(_WIN32)
|
||||
config_directory = common_get_env("APPDATA");
|
||||
if (config_directory.empty()) {
|
||||
throw std::runtime_error("Failed to find %APPDATA% directory");
|
||||
}
|
||||
#elif defined(__EMSCRIPTEN__)
|
||||
// caller decides what to do when there is no config directory
|
||||
throw std::runtime_error("not implemented on this platform");
|
||||
#else
|
||||
config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
|
||||
if (config_directory.empty()) {
|
||||
config_directory = get_home_directory() / ".config";
|
||||
}
|
||||
# error Unknown architecture
|
||||
#endif
|
||||
return config_directory / "llama.cpp";
|
||||
config_directory = ensure_trailing_slash(config_directory);
|
||||
config_directory += "llama.cpp";
|
||||
return ensure_trailing_slash(config_directory);
|
||||
}
|
||||
|
||||
std::filesystem::path fs_get_cache_file(const std::string & filename) {
|
||||
std::string fs_get_cache_file(const std::string & filename) {
|
||||
GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
|
||||
const std::filesystem::path cache_directory = fs_get_cache_directory();
|
||||
std::error_code ec;
|
||||
common_create_directories(cache_directory, ec);
|
||||
if (ec) {
|
||||
throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
|
||||
std::string cache_directory = fs_get_cache_directory();
|
||||
const bool success = fs_create_directory_with_parents(cache_directory);
|
||||
if (!success) {
|
||||
throw std::runtime_error("failed to create cache directory: " + cache_directory);
|
||||
}
|
||||
return cache_directory / std::filesystem::u8path(filename);
|
||||
return cache_directory + filename;
|
||||
}
|
||||
|
||||
std::vector<common_file_info> fs_list(const std::string & path, bool include_directories) {
|
||||
std::vector<common_file_info> files;
|
||||
if (path.empty()) return files;
|
||||
|
||||
std::filesystem::path dir(path);
|
||||
if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) {
|
||||
return files;
|
||||
}
|
||||
|
||||
for (const auto & entry : std::filesystem::directory_iterator(dir)) {
|
||||
try {
|
||||
// Only include regular files (skip directories)
|
||||
const auto & p = entry.path();
|
||||
if (std::filesystem::is_regular_file(p)) {
|
||||
common_file_info info;
|
||||
info.path = p.string();
|
||||
info.name = p.filename().string();
|
||||
info.is_dir = false;
|
||||
try {
|
||||
info.size = static_cast<size_t>(std::filesystem::file_size(p));
|
||||
} catch (const std::filesystem::filesystem_error &) {
|
||||
info.size = 0;
|
||||
}
|
||||
files.push_back(std::move(info));
|
||||
} else if (include_directories && std::filesystem::is_directory(p)) {
|
||||
common_file_info info;
|
||||
info.path = p.string();
|
||||
info.name = p.filename().string();
|
||||
info.size = 0; // Directories have no size
|
||||
info.is_dir = true;
|
||||
files.push_back(std::move(info));
|
||||
}
|
||||
} catch (const std::filesystem::filesystem_error &) {
|
||||
// skip entries we cannot inspect
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) {
|
||||
#ifdef _WIN32
|
||||
int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);
|
||||
if (!wlen) { return std::ifstream(); }
|
||||
std::vector<wchar_t> wfname(wlen);
|
||||
(void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);
|
||||
return std::ifstream(wfname.data(), mode);
|
||||
#else
|
||||
return std::ifstream(fname, mode);
|
||||
#endif
|
||||
}
|
||||
|
||||
//
|
||||
// TTY utils
|
||||
//
|
||||
|
||||
bool common_is_tty(FILE * file) {
|
||||
#if defined(_WIN32)
|
||||
return _isatty(_fileno(file));
|
||||
#else
|
||||
return isatty(fileno(file));
|
||||
#endif
|
||||
}
|
||||
|
||||
bool tty_can_use_colors() {
|
||||
// Check NO_COLOR environment variable (https://no-color.org/)
|
||||
if (const char * no_color = std::getenv("NO_COLOR")) {
|
||||
@@ -1063,21 +1199,10 @@ bool tty_can_use_colors() {
|
||||
|
||||
// Check if stdout and stderr are connected to a terminal
|
||||
// We check both because log messages can go to either
|
||||
return common_is_tty(stdout) || common_is_tty(stderr);
|
||||
}
|
||||
bool stdout_is_tty = isatty(fileno(stdout));
|
||||
bool stderr_is_tty = isatty(fileno(stderr));
|
||||
|
||||
bool tty_enable_ansi() {
|
||||
#if defined(_WIN32)
|
||||
// a Windows console renders ANSI sequences only in virtual terminal mode, pipes and files take them as is
|
||||
for (DWORD id : { STD_OUTPUT_HANDLE, STD_ERROR_HANDLE }) {
|
||||
HANDLE h = GetStdHandle(id);
|
||||
DWORD mode = 0;
|
||||
if (GetConsoleMode(h, &mode) && !SetConsoleMode(h, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return true;
|
||||
return stdout_is_tty || stderr_is_tty;
|
||||
}
|
||||
|
||||
//
|
||||
@@ -1162,71 +1287,6 @@ struct common_init_result::impl {
|
||||
std::vector<llama_sampler_seq_config> samplers_seq_config;
|
||||
};
|
||||
|
||||
static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NAMES = {
|
||||
{ COMMON_DECISION_TYPE_OPENJEV, "openjev" },
|
||||
{ COMMON_DECISION_TYPE_LEV, "lev" },
|
||||
{ COMMON_DECISION_TYPE_KEV, "kev" },
|
||||
{ COMMON_DECISION_TYPE_NIMBLE, "nimble" },
|
||||
{ COMMON_DECISION_TYPE_LAYA, "laya" },
|
||||
{ COMMON_DECISION_TYPE_CLEF, "clef" },
|
||||
};
|
||||
|
||||
static common_decision_type common_decision_type_from_string(const std::string & str) {
|
||||
for (const auto & pair : COMMON_DECISION_TYPE_NAMES) {
|
||||
if (pair.second == str) {
|
||||
return pair.first;
|
||||
}
|
||||
}
|
||||
return COMMON_DECISION_TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model) {
|
||||
char buf[64];
|
||||
if (llama_model_meta_val_str(model, "general.architecture", buf, sizeof(buf)) < 0) {
|
||||
return COMMON_DECISION_TYPE_NONE;
|
||||
}
|
||||
const std::string key = std::string(buf) + ".decision.type";
|
||||
if (llama_model_meta_val_str(model, key.c_str(), buf, sizeof(buf)) < 0) {
|
||||
return COMMON_DECISION_TYPE_NONE;
|
||||
}
|
||||
return common_decision_type_from_string(buf);
|
||||
}
|
||||
|
||||
common_decision_type common_get_decision_type(const std::string & fname) {
|
||||
struct gguf_init_params gguf_params = {
|
||||
/* .no_alloc = */ true,
|
||||
/* .ctx = */ nullptr,
|
||||
};
|
||||
|
||||
gguf_context_ptr gguf_ctx(gguf_init_from_file(fname.c_str(), gguf_params));
|
||||
if (!gguf_ctx) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // missing or unreadable file
|
||||
}
|
||||
|
||||
std::string arch;
|
||||
const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture");
|
||||
if (arch_id < 0) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // no architecture in the metadata
|
||||
}
|
||||
if (gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
|
||||
}
|
||||
arch = gguf_get_val_str(gguf_ctx.get(), arch_id);
|
||||
if (arch.empty()) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
const std::string key = arch + ".decision.type";
|
||||
const int64_t type_id = gguf_find_key(gguf_ctx.get(), key.c_str());
|
||||
if (type_id < 0) {
|
||||
return COMMON_DECISION_TYPE_NONE;
|
||||
}
|
||||
if (gguf_get_kv_type(gguf_ctx.get(), type_id) != GGUF_TYPE_STRING) {
|
||||
return COMMON_DECISION_TYPE_UNKNOWN; // malformed metadata
|
||||
}
|
||||
return common_decision_type_from_string(gguf_get_val_str(gguf_ctx.get(), type_id));
|
||||
}
|
||||
|
||||
common_init_result::common_init_result(common_params & params, bool model_only) :
|
||||
pimpl(new impl{}) {
|
||||
auto mparams = common_model_params_to_llama(params);
|
||||
@@ -1279,29 +1339,6 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
|
||||
// these decision models return a score for each token via the embeddings output
|
||||
// TODO: maybe improve this in the future
|
||||
const auto decision_type = common_get_decision_type(model);
|
||||
if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF) {
|
||||
params.embedding = true;
|
||||
params.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
||||
|
||||
cparams.embeddings = true;
|
||||
cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
||||
cparams.n_outputs_max = cparams.n_batch;
|
||||
cparams.n_outputs_max_per_seq = 1;
|
||||
|
||||
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
|
||||
}
|
||||
|
||||
// embeddings need the whole batch in one ubatch, so n_batch must not be larger than n_ubatch
|
||||
// (server.cpp does this check for --embedding, but before the model is loaded)
|
||||
if (cparams.embeddings && cparams.n_batch > cparams.n_ubatch) {
|
||||
LOG_WRN("embeddings enabled: setting n_batch = n_ubatch = %u\n", cparams.n_ubatch);
|
||||
cparams.n_batch = cparams.n_ubatch;
|
||||
params.n_batch = params.n_ubatch;
|
||||
}
|
||||
|
||||
// load and optionally apply lora adapters
|
||||
for (auto & la : params.lora_adapters) {
|
||||
llama_adapter_lora_ptr lora;
|
||||
@@ -1490,8 +1527,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
}
|
||||
|
||||
if (llama_model_has_encoder(model)) {
|
||||
common_batch batch = common_batch_get_one(lctx, tmp);
|
||||
llama_process(lctx, LLAMA_PROCESS_TYPE_ENCODE, batch.get());
|
||||
llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size()));
|
||||
llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
|
||||
if (decoder_start_token_id == LLAMA_TOKEN_NULL) {
|
||||
decoder_start_token_id = bos;
|
||||
@@ -1500,9 +1536,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
tmp.push_back(decoder_start_token_id);
|
||||
}
|
||||
if (llama_model_has_decoder(model)) {
|
||||
tmp.resize(std::min(tmp.size(), (size_t) params.n_batch));
|
||||
common_batch batch = common_batch_get_one(lctx, tmp);
|
||||
llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
|
||||
llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch)));
|
||||
}
|
||||
llama_memory_clear(llama_get_memory(lctx), true);
|
||||
llama_synchronize(lctx);
|
||||
@@ -1552,11 +1586,6 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
|
||||
return COMMON_CONTEXT_SEQ_RM_TYPE_NO;
|
||||
}
|
||||
|
||||
if (llama_n_rs_seq(ctx) > 0) {
|
||||
COM_TRC("%s", "the context supports bounded partial sequence removal\n");
|
||||
return COMMON_CONTEXT_SEQ_RM_TYPE_RS;
|
||||
}
|
||||
|
||||
common_context_seq_rm_type res = COMMON_CONTEXT_SEQ_RM_TYPE_PART;
|
||||
|
||||
llama_memory_clear(mem, true);
|
||||
@@ -1566,17 +1595,19 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
|
||||
tmp.push_back(0);
|
||||
tmp.push_back(0);
|
||||
|
||||
int ret;
|
||||
{
|
||||
common_batch batch = common_batch_get_one(ctx, tmp);
|
||||
ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
|
||||
}
|
||||
int ret = llama_decode(ctx, llama_batch_get_one(tmp.data(), tmp.size()));
|
||||
if (ret != 0) {
|
||||
COM_ERR("llama_process() failed: %d\n", ret);
|
||||
COM_ERR("llama_decode() failed: %d\n", ret);
|
||||
res = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
|
||||
goto done;
|
||||
}
|
||||
|
||||
if (llama_n_rs_seq(ctx) > 0) {
|
||||
COM_TRC("%s", "the context supports bounded partial sequence removal\n");
|
||||
res = COMMON_CONTEXT_SEQ_RM_TYPE_RS;
|
||||
goto done;
|
||||
}
|
||||
|
||||
// try to remove the last tokens
|
||||
if (!llama_memory_seq_rm(mem, 0, 1, -1)) {
|
||||
COM_TRC("%s", "the context does not support partial sequence removal\n");
|
||||
@@ -1657,7 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
|
||||
mparams.main_gpu = params.main_gpu;
|
||||
mparams.split_mode = params.split_mode;
|
||||
mparams.load_mode = params.load_mode;
|
||||
mparams.lazy_mode = params.lazy_mode;
|
||||
mparams.tensor_read_lazy = params.tensor_read_lazy;
|
||||
mparams.tensor_split = params.tensor_split;
|
||||
mparams.check_tensors = params.check_tensors;
|
||||
mparams.use_extra_bufts = !params.no_extra_bufts;
|
||||
@@ -1680,7 +1711,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
|
||||
mparams.progress_callback = params.load_progress_callback;
|
||||
mparams.progress_callback_user_data = params.load_progress_callback_user_data;
|
||||
mparams.no_alloc = params.no_alloc;
|
||||
mparams.load_mtp = params.load_mtp || std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
|
||||
|
||||
return mparams;
|
||||
}
|
||||
@@ -1796,6 +1827,33 @@ void common_threadpools::init(llama_context * ctx, const common_params & params)
|
||||
llama_attach_threadpool(ctx, threadpool, threadpool_batch);
|
||||
}
|
||||
|
||||
//
|
||||
// Batch utils
|
||||
//
|
||||
|
||||
void common_batch_clear(struct llama_batch & batch) {
|
||||
batch.n_tokens = 0;
|
||||
}
|
||||
|
||||
void common_batch_add(
|
||||
struct llama_batch & batch,
|
||||
llama_token id,
|
||||
llama_pos pos,
|
||||
const std::vector<llama_seq_id> & seq_ids,
|
||||
bool logits) {
|
||||
GGML_ASSERT(batch.seq_id[batch.n_tokens] && "llama_batch size exceeded");
|
||||
|
||||
batch.token [batch.n_tokens] = id;
|
||||
batch.pos [batch.n_tokens] = pos;
|
||||
batch.n_seq_id[batch.n_tokens] = seq_ids.size();
|
||||
for (size_t i = 0; i < seq_ids.size(); ++i) {
|
||||
batch.seq_id[batch.n_tokens][i] = seq_ids[i];
|
||||
}
|
||||
batch.logits [batch.n_tokens] = logits;
|
||||
|
||||
batch.n_tokens++;
|
||||
}
|
||||
|
||||
//
|
||||
// Vocab utils
|
||||
//
|
||||
@@ -2132,140 +2190,18 @@ float lr_opt::get_lr(float epoch) const {
|
||||
}
|
||||
|
||||
bool common_replay_last_token(struct llama_context * ctx, llama_token last_token, int32_t pos) {
|
||||
common_batch batch(ctx);
|
||||
batch.add(last_token, pos, 0, true);
|
||||
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
llama_batch batch = llama_batch_get_one(&last_token, 1);
|
||||
batch.pos = &pos;
|
||||
if (llama_decode(ctx, batch)) {
|
||||
LOG_ERR("%s: failed to replay last token\n", __func__);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
common_batch::common_batch(llama_context * ctx) : batch(llama_batch_ext_init(ctx)) {
|
||||
const auto rope_type = llama_model_rope_type(llama_get_model(ctx));
|
||||
n_pos = rope_type == LLAMA_ROPE_TYPE_MROPE || rope_type == LLAMA_ROPE_TYPE_IMROPE ? GGML_MROPE_SECTIONS : 1;
|
||||
}
|
||||
|
||||
void common_batch::clear() {
|
||||
tokens.clear();
|
||||
}
|
||||
|
||||
int32_t common_batch::add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output) {
|
||||
tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, {} });
|
||||
return size() - 1;
|
||||
}
|
||||
|
||||
int32_t common_batch::add(llama_token id, llama_pos pos, const std::vector<llama_seq_id> & seq_ids, bool output) {
|
||||
GGML_ASSERT(!seq_ids.empty());
|
||||
|
||||
const int32_t idx = add(id, pos, seq_ids[0], output);
|
||||
for (size_t s = 1; s < seq_ids.size(); ++s) {
|
||||
add_seq(idx, seq_ids[s]);
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
|
||||
bool common_batch::add_seq(int32_t idx, llama_seq_id seq_id) {
|
||||
if (idx < 0 || idx >= size()) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].seq_ids_extra.push_back(seq_id);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool common_batch::set_output(int32_t idx, bool value) {
|
||||
if (idx < 0 || idx >= size()) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].output = value;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool common_batch::set_embd(int32_t idx, llama_embd embd) {
|
||||
if (idx < 0 || idx >= size() || tokens[idx].embd.data != nullptr) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].embd = embd;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
|
||||
token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, {} };
|
||||
for (int32_t j = 0; j < n_pos; ++j) {
|
||||
t.pos[j] = pos[j];
|
||||
}
|
||||
tokens.push_back(t);
|
||||
return size() - 1;
|
||||
}
|
||||
|
||||
llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
GGML_ASSERT(batch && "common_batch was not initialized with a context");
|
||||
GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size());
|
||||
|
||||
llama_batch_ext * res = batch.get();
|
||||
llama_batch_ext_clear(res);
|
||||
|
||||
for (int32_t i = off; i < off + n; ++i) {
|
||||
const token & t = tokens[i];
|
||||
|
||||
int32_t idx;
|
||||
if (t.id != LLAMA_TOKEN_NULL) {
|
||||
idx = llama_batch_ext_add_token(res, t.seq_id, t.id);
|
||||
if (idx < 0) {
|
||||
GGML_ABORT("%s: failed to add token %d at index %d (error %d, n = %d)\n", __func__, t.id, i, idx, n);
|
||||
}
|
||||
llama_batch_ext_set_pos(res, idx, t.pos.data());
|
||||
if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
|
||||
GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
|
||||
}
|
||||
} else {
|
||||
idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
|
||||
if (idx < 0) {
|
||||
GGML_ABORT("%s: failed to add embedding at index %d (error %d, n = %d)\n", __func__, i, idx, n);
|
||||
}
|
||||
llama_batch_ext_set_pos(res, idx, t.pos.data());
|
||||
}
|
||||
GGML_ASSERT(idx == i - off);
|
||||
|
||||
for (const llama_seq_id seq_id : t.seq_ids_extra) {
|
||||
if (!llama_batch_ext_add_seq(res, idx, seq_id)) {
|
||||
GGML_ABORT("%s: failed to add seq %d to the entry at index %d\n", __func__, seq_id, i);
|
||||
}
|
||||
}
|
||||
if (t.output) {
|
||||
llama_batch_ext_set_output_logits(res, idx, true);
|
||||
}
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
common_batch common_batch_get_one(llama_context * ctx, const llama_token * tokens, int32_t n_tokens) {
|
||||
common_batch batch(ctx);
|
||||
|
||||
auto mem = llama_get_memory(ctx);
|
||||
llama_pos pos = llama_memory_seq_pos_max(mem, 0) + 1; // -1 + 1 == 0 when the memory is empty
|
||||
|
||||
for (int32_t i = 0; i < n_tokens; ++i) {
|
||||
const bool output = i == n_tokens - 1;
|
||||
batch.add(tokens[i], pos, 0, output);
|
||||
pos++;
|
||||
}
|
||||
|
||||
return batch;
|
||||
}
|
||||
|
||||
common_batch common_batch_get_one(llama_context * ctx, const llama_tokens & tokens) {
|
||||
return common_batch_get_one(ctx, tokens.data(), (int32_t) tokens.size());
|
||||
}
|
||||
|
||||
bool common_prompt_batch_decode(
|
||||
struct llama_context * ctx,
|
||||
const llama_tokens & all_tokens,
|
||||
const std::vector<llama_token> & all_tokens,
|
||||
int n_new,
|
||||
int & n_past,
|
||||
int n_batch,
|
||||
@@ -2286,9 +2222,7 @@ bool common_prompt_batch_decode(
|
||||
// Memory implementations in recurrent/hybrid models don't support removing tokens from their
|
||||
// memory, so we can't just remove the last token from the memory and replay the last token which
|
||||
// is the reason for this logic.
|
||||
llama_tokens prefix_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_tokens_before_last);
|
||||
common_batch batch_prefix = common_batch_get_one(ctx, prefix_tokens);
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_prefix.get())) {
|
||||
if (llama_decode(ctx, llama_batch_get_one(const_cast<llama_token*>(all_tokens.data() + offset), n_tokens_before_last))) {
|
||||
COM_ERR("%s", "failed to eval\n");
|
||||
return false;
|
||||
}
|
||||
@@ -2297,18 +2231,18 @@ bool common_prompt_batch_decode(
|
||||
llama_state_save_file(ctx, state_path.data(), all_tokens.data(), all_tokens.size());
|
||||
COM_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size());
|
||||
|
||||
common_batch batch_last(ctx);
|
||||
batch_last.add(all_tokens.back(), n_past, 0, true);
|
||||
llama_token last_token = all_tokens.back();
|
||||
llama_batch batch = llama_batch_get_one(&last_token, 1);
|
||||
int32_t pos = n_past;
|
||||
batch.pos = &pos;
|
||||
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_last.get())) {
|
||||
if (llama_decode(ctx, batch)) {
|
||||
COM_ERR("%s", "failed to eval last token\n");
|
||||
return false;
|
||||
}
|
||||
n_past++;
|
||||
} else {
|
||||
llama_tokens new_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_new);
|
||||
common_batch batch = common_batch_get_one(ctx, new_tokens);
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
if (llama_decode(ctx, llama_batch_get_one(const_cast<llama_token*>(all_tokens.data() + offset), n_new))) {
|
||||
COM_ERR("%s", "failed to eval\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
+30
-124
@@ -8,7 +8,6 @@
|
||||
#include "ggml.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include <array>
|
||||
#include <list>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
@@ -17,9 +16,7 @@
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <cstdio>
|
||||
|
||||
#if defined(_WIN32) && !defined(_WIN32_WINNT)
|
||||
#define _WIN32_WINNT 0x0A00
|
||||
@@ -273,7 +270,7 @@ struct common_params_sampling {
|
||||
COMMON_SAMPLER_TYPE_TEMPERATURE,
|
||||
};
|
||||
|
||||
common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls)
|
||||
common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls)
|
||||
bool grammar_lazy = false;
|
||||
std::vector<common_grammar_trigger> grammar_triggers; // optional triggers (for lazy grammars)
|
||||
std::set<llama_token> preserved_tokens;
|
||||
@@ -334,8 +331,6 @@ struct common_params_speculative_draft {
|
||||
|
||||
bool backend_sampling = true; // offload draft sampling to the backend (default: on)
|
||||
|
||||
bool probabilistic = false; // sample the draft and verify by rejection, instead of argmax and match
|
||||
|
||||
common_params_model mparams;
|
||||
|
||||
llama_context * ctx_tgt = nullptr;
|
||||
@@ -488,7 +483,7 @@ struct common_params {
|
||||
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
|
||||
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model
|
||||
|
||||
enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch
|
||||
enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch
|
||||
|
||||
common_cpu_params cpuparams;
|
||||
common_cpu_params cpuparams_batch;
|
||||
@@ -586,7 +581,6 @@ struct common_params {
|
||||
bool no_op_offload = false; // globally disable offload host tensor operations to device
|
||||
bool no_extra_bufts = false; // disable extra buffer types (used for weight repacking)
|
||||
bool no_host = false; // bypass host buffer allowing extra buffers to be used
|
||||
bool load_mtp = false; // load MTP/NextN layers
|
||||
|
||||
bool single_turn = false; // single turn chat conversation
|
||||
|
||||
@@ -637,10 +631,10 @@ struct common_params {
|
||||
int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints
|
||||
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
|
||||
|
||||
std::string hostname = "127.0.0.1";
|
||||
std::string public_path = ""; // NOLINT
|
||||
std::string api_prefix = ""; // NOLINT
|
||||
std::string chat_template = ""; // NOLINT
|
||||
std::vector<std::string> hostnames = {"127.0.0.1"};
|
||||
bool use_jinja = true; // NOLINT
|
||||
|
||||
// server CORS params
|
||||
@@ -663,7 +657,6 @@ struct common_params {
|
||||
std::string ssl_file_cert = ""; // NOLINT
|
||||
|
||||
std::map<std::string, std::string> default_template_kwargs;
|
||||
bool preserve_reasoning_specified = false;
|
||||
|
||||
// CLI params
|
||||
std::string server_base; // if set, connect to this server instead of starting a new one
|
||||
@@ -725,11 +718,10 @@ struct common_params {
|
||||
int32_t i_chunk = 0; // start processing from this chunk
|
||||
int8_t imat_dat = 0; // whether the legacy imatrix.dat format should be output (gguf <= 0 < dat)
|
||||
|
||||
bool process_output = false; // collect data for the output tensor
|
||||
bool compute_ppl = true; // whether to compute perplexity
|
||||
bool show_statistics = false; // show imatrix statistics per tensor
|
||||
bool activation_statistics = false; // generate data to calculate activation based statistics
|
||||
bool parse_special = false; // whether to parse special tokens during imatrix tokenization
|
||||
bool process_output = false; // collect data for the output tensor
|
||||
bool compute_ppl = true; // whether to compute perplexity
|
||||
bool show_statistics = false; // show imatrix statistics per tensor
|
||||
bool parse_special = false; // whether to parse special tokens during imatrix tokenization
|
||||
|
||||
// cvector-generator params
|
||||
int n_pca_batch = 100;
|
||||
@@ -815,9 +807,7 @@ static std::vector<T> string_split(const std::string & str, char delim) {
|
||||
while (std::getline(str_stream, token, delim)) {
|
||||
T value;
|
||||
std::istringstream token_stream(token);
|
||||
if (!(token_stream >> value)) {
|
||||
throw std::invalid_argument("invalid value: \"" + token + "\"");
|
||||
}
|
||||
token_stream >> value;
|
||||
values.push_back(value);
|
||||
}
|
||||
return values;
|
||||
@@ -885,21 +875,10 @@ void string_process_escapes(std::string & input);
|
||||
std::string string_from(bool value);
|
||||
std::string string_from(const std::vector<int> & values);
|
||||
std::string string_from(const struct llama_context * ctx, const std::vector<llama_token> & tokens);
|
||||
std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch);
|
||||
|
||||
bool glob_match(const std::string & pattern, const std::string & str);
|
||||
|
||||
//
|
||||
// Unicode utils
|
||||
//
|
||||
|
||||
#ifdef _WIN32
|
||||
std::wstring utf8_to_wstring(const std::string & str);
|
||||
std::string wstring_to_utf8(const std::wstring & str);
|
||||
#endif
|
||||
|
||||
// returns the path as a UTF-8 string, preserving its separators
|
||||
std::string fs_path_to_utf8(const std::filesystem::path & path);
|
||||
|
||||
//
|
||||
// Environment utils
|
||||
//
|
||||
@@ -909,30 +888,28 @@ std::string fs_path_to_utf8(const std::filesystem::path & path);
|
||||
std::string common_get_env(const std::string & name);
|
||||
void common_set_env(const std::string & name, const std::string & value);
|
||||
|
||||
// reads a path from the environment, an unset variable gives an empty path
|
||||
std::filesystem::path common_get_path_from_env(const std::string & name);
|
||||
|
||||
//
|
||||
// Filesystem utils
|
||||
//
|
||||
|
||||
bool fs_validate_filename(const std::string & filename, bool allow_subdirs = false);
|
||||
bool fs_create_directory_with_parents(const std::string & path);
|
||||
bool fs_is_directory(const std::string & path);
|
||||
|
||||
// some old libstdc++ versions don't follow symlinks here, so adding a trailing "/" fixes it: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101510
|
||||
inline bool common_create_directories(const std::filesystem::path & path, std::error_code & ec) {
|
||||
#if defined(__linux__)
|
||||
return std::filesystem::create_directories(path / "", ec);
|
||||
#else
|
||||
return std::filesystem::create_directories(path, ec);
|
||||
#endif
|
||||
}
|
||||
std::string fs_get_cache_directory();
|
||||
std::string fs_get_cache_file(const std::string & filename);
|
||||
std::string fs_get_config_directory();
|
||||
|
||||
std::filesystem::path fs_get_cache_directory();
|
||||
std::filesystem::path fs_get_cache_file(const std::string & filename);
|
||||
std::filesystem::path fs_get_config_directory();
|
||||
struct common_file_info {
|
||||
std::string path;
|
||||
std::string name;
|
||||
size_t size = 0; // in bytes
|
||||
bool is_dir = false;
|
||||
};
|
||||
std::vector<common_file_info> fs_list(const std::string & path, bool include_directories);
|
||||
|
||||
void fs_write_atomic(const std::filesystem::path & path, const std::string & data);
|
||||
// fs open, also handle UTF8 on Windows
|
||||
std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode);
|
||||
|
||||
//
|
||||
// TTY utils
|
||||
@@ -940,10 +917,6 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat
|
||||
|
||||
// Auto-detect if colors can be enabled based on terminal and environment
|
||||
bool tty_can_use_colors();
|
||||
bool tty_enable_ansi(); // false when stdout or stderr is a console that cannot render ANSI sequences
|
||||
|
||||
// Check if the given file is attached to a terminal
|
||||
bool common_is_tty(FILE * file);
|
||||
|
||||
//
|
||||
// Model utils
|
||||
@@ -951,24 +924,6 @@ bool common_is_tty(FILE * file);
|
||||
|
||||
struct common_sampler;
|
||||
|
||||
// typed decision models, see "<arch>.decision.type" in the model metadata
|
||||
enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_NONE, // not a decision model
|
||||
COMMON_DECISION_TYPE_OPENJEV, // logits of one label token per option, read at the last prompt token
|
||||
COMMON_DECISION_TYPE_LEV, // same as openjev, noul is read from a rating scale
|
||||
COMMON_DECISION_TYPE_KEV, // dot product of the hidden states of the last token and of one end token per option
|
||||
COMMON_DECISION_TYPE_NIMBLE, // same as openjev, the prompt lists all the questions of the request
|
||||
COMMON_DECISION_TYPE_LAYA, // score of one marker token per option, read from the embeddings output
|
||||
COMMON_DECISION_TYPE_CLEF, // all questions in one prompt, score of option i read from the embeddings output at row i
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model);
|
||||
|
||||
// same as above, but reads a GGUF file; it does not load the model
|
||||
// returns COMMON_DECISION_TYPE_UNKNOWN if the file is missing, unreadable, or invalid
|
||||
common_decision_type common_get_decision_type(const std::string & fname);
|
||||
|
||||
// note: defines the model, context, samplers, ets. lifetimes
|
||||
struct common_init_result {
|
||||
common_init_result(common_params & params, bool model_only = false);
|
||||
@@ -1056,63 +1011,14 @@ struct common_memory {
|
||||
// Batch utils
|
||||
//
|
||||
|
||||
// wrapper around llama_batch_ext that provide getter functions for downstream code
|
||||
// entries can exceed n_batch, use get_sub_batch() to decode them in chunks
|
||||
struct common_batch {
|
||||
struct token {
|
||||
llama_token id;
|
||||
std::array<llama_pos, GGML_MROPE_SECTIONS> pos; // only pos[0] is used for text tokens
|
||||
llama_seq_id seq_id; // the first sequence id, see add_seq()
|
||||
bool output;
|
||||
llama_embd embd; // non-owning view of the data passed to add_embd()/set_embd(), data == NULL if none
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
};
|
||||
void common_batch_clear(struct llama_batch & batch);
|
||||
|
||||
std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
|
||||
llama_batch_ext_ptr batch;
|
||||
|
||||
int32_t n_pos = 1; // positions per embedding entry, GGML_MROPE_SECTIONS for MROPE/IMROPE
|
||||
|
||||
common_batch() = default;
|
||||
common_batch(struct llama_context * ctx);
|
||||
|
||||
llama_batch_ext * get() { return get_sub_batch(0, size()); }
|
||||
|
||||
// render entries [off, off + n) into batch, the result is overwritten by the next call
|
||||
llama_batch_ext * get_sub_batch(int32_t off, int32_t n);
|
||||
|
||||
// content type of the batch, all entries carry the same combination
|
||||
bool has_token() const { return !tokens.empty() && tokens[0].id != LLAMA_TOKEN_NULL; }
|
||||
bool has_embd () const { return !tokens.empty() && tokens[0].embd.data != nullptr; }
|
||||
|
||||
void clear();
|
||||
|
||||
// returns the batch index
|
||||
int32_t add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output);
|
||||
|
||||
// same, with the entry shared by all seq_ids (must not be empty)
|
||||
int32_t add(llama_token id, llama_pos pos, const std::vector<llama_seq_id> & seq_ids, bool output);
|
||||
|
||||
// add the entry at idx to another sequence, tokens[idx].seq_id keeps the first one
|
||||
bool add_seq(int32_t idx, llama_seq_id seq_id);
|
||||
|
||||
bool set_output(int32_t idx, bool value);
|
||||
|
||||
// attach a token embedding to the entry at idx, can only be set once per entry
|
||||
bool set_embd(int32_t idx, llama_embd embd);
|
||||
|
||||
// add an embedding-only entry (no token id)
|
||||
// pos points to n_pos positions
|
||||
int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
|
||||
|
||||
int32_t size() const { return (int32_t) tokens.size(); }
|
||||
};
|
||||
|
||||
// create a single-sequence batch from a list of tokens
|
||||
// positions continue from the memory, last token always have output_logits set to true
|
||||
common_batch common_batch_get_one(struct llama_context * ctx, const llama_token * tokens, int32_t n_tokens);
|
||||
common_batch common_batch_get_one(struct llama_context * ctx, const llama_tokens & tokens);
|
||||
void common_batch_add(
|
||||
struct llama_batch & batch,
|
||||
llama_token id,
|
||||
llama_pos pos,
|
||||
const std::vector<llama_seq_id> & seq_ids,
|
||||
bool logits);
|
||||
|
||||
// decodes a single batch of tokens for a prompt and manages session tokens
|
||||
//
|
||||
@@ -1121,7 +1027,7 @@ common_batch common_batch_get_one(struct llama_context * ctx, const llama_tokens
|
||||
// tokens from memory, so this approach works across all model architectures.
|
||||
bool common_prompt_batch_decode(
|
||||
struct llama_context * ctx,
|
||||
const llama_tokens & all_tokens,
|
||||
const std::vector<llama_token> & all_tokens,
|
||||
int n_new,
|
||||
int & n_past,
|
||||
int n_batch,
|
||||
|
||||
+5
-4
@@ -1,5 +1,4 @@
|
||||
#include "console.h"
|
||||
#include "common.h"
|
||||
#include "log.h"
|
||||
#include <vector>
|
||||
#include <iostream>
|
||||
@@ -1019,7 +1018,6 @@ namespace console {
|
||||
line.clear();
|
||||
pop_cursor();
|
||||
}
|
||||
line += '\n';
|
||||
has_more = false;
|
||||
}
|
||||
} else {
|
||||
@@ -1051,10 +1049,13 @@ namespace console {
|
||||
if (!std::getline(std::wcin, wline)) {
|
||||
// Input stream is bad or EOF received
|
||||
line.clear();
|
||||
GenerateConsoleCtrlEvent(CTRL_C_EVENT, 0);
|
||||
return false;
|
||||
}
|
||||
|
||||
line = wstring_to_utf8(wline);
|
||||
int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), NULL, 0, NULL, NULL);
|
||||
line.resize(size_needed);
|
||||
WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), &line[0], size_needed, NULL, NULL);
|
||||
#else
|
||||
if (!std::getline(std::cin, line)) {
|
||||
// Input stream is bad or EOF received
|
||||
@@ -1065,7 +1066,7 @@ namespace console {
|
||||
if (!line.empty()) {
|
||||
char last = line.back();
|
||||
if (last == '/') { // Always return control on '/' symbol
|
||||
line.back() = '\n';
|
||||
line.pop_back();
|
||||
return false;
|
||||
}
|
||||
if (last == '\\') { // '\\' changes the default action
|
||||
|
||||
+46
-11
@@ -35,13 +35,50 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// isatty
|
||||
#if defined(_WIN32)
|
||||
#include <io.h>
|
||||
#else
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
//
|
||||
// downloader
|
||||
//
|
||||
|
||||
// validate repo name format: owner/repo
|
||||
static void write_file(const std::string & fname, const std::string & content) {
|
||||
const std::string fname_tmp = fname + ".tmp";
|
||||
std::ofstream file(fname_tmp);
|
||||
if (!file) {
|
||||
throw std::runtime_error(string_format("error: failed to open file '%s'\n", fname.c_str()));
|
||||
}
|
||||
|
||||
try {
|
||||
file << content;
|
||||
file.close();
|
||||
|
||||
// Makes write atomic
|
||||
if (rename(fname_tmp.c_str(), fname.c_str()) != 0) {
|
||||
LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, fname_tmp.c_str(), fname.c_str());
|
||||
// If rename fails, try to delete the temporary file
|
||||
if (remove(fname_tmp.c_str()) != 0) {
|
||||
LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, fname_tmp.c_str());
|
||||
}
|
||||
}
|
||||
} catch (...) {
|
||||
// If anything fails, try to delete the temporary file
|
||||
if (remove(fname_tmp.c_str()) != 0) {
|
||||
LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, fname_tmp.c_str());
|
||||
}
|
||||
|
||||
throw std::runtime_error(string_format("error: failed to write file '%s'\n", fname.c_str()));
|
||||
}
|
||||
}
|
||||
|
||||
static void write_etag(const std::string & path, const std::string & etag) {
|
||||
const std::string etag_path = path + ".etag";
|
||||
fs_write_atomic(std::filesystem::u8path(etag_path), etag);
|
||||
write_file(etag_path, etag);
|
||||
LOG_DBG("%s: file etag saved: %s\n", __func__, etag_path.c_str());
|
||||
}
|
||||
|
||||
@@ -90,7 +127,11 @@ class ProgressBar : public common_download_callback {
|
||||
}
|
||||
|
||||
static bool is_output_a_tty() {
|
||||
return common_is_tty(stdout);
|
||||
#if defined(_WIN32)
|
||||
return _isatty(_fileno(stdout));
|
||||
#else
|
||||
return isatty(1);
|
||||
#endif
|
||||
}
|
||||
|
||||
public:
|
||||
@@ -233,12 +274,6 @@ static bool common_pull_file(httplib::Client & cli,
|
||||
return false;
|
||||
}
|
||||
|
||||
ofs.close();
|
||||
if (!ofs) {
|
||||
LOG_ERR("%s: error closing file: %s\n", __func__, path_tmp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -251,7 +286,7 @@ static int common_download_file_single_online(const std::string & url,
|
||||
static const int max_attempts = 3;
|
||||
static const int retry_delay_seconds = 2;
|
||||
|
||||
const bool file_exists = std::filesystem::exists(std::filesystem::u8path(path));
|
||||
const bool file_exists = std::filesystem::exists(path);
|
||||
|
||||
if (file_exists && skip_etag) {
|
||||
LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
|
||||
@@ -442,7 +477,7 @@ int common_download_file_single(const std::string & url,
|
||||
return common_download_file_single_online(url, path, online_opts, skip_etag);
|
||||
}
|
||||
|
||||
if (!std::filesystem::exists(std::filesystem::u8path(path))) {
|
||||
if (!std::filesystem::exists(path)) {
|
||||
LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str());
|
||||
return -1;
|
||||
}
|
||||
@@ -908,7 +943,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
|
||||
std::string model_filename = repo;
|
||||
std::replace(model_filename.begin(), model_filename.end(), '/', '_');
|
||||
model_filename += "_" + tag + ".gguf";
|
||||
std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
|
||||
std::string local_path = fs_get_cache_file(model_filename);
|
||||
|
||||
const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
|
||||
common_download_opts opts;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#include "fit.h"
|
||||
|
||||
#include "json.h"
|
||||
#include "log.h"
|
||||
|
||||
#include "../src/llama-ext.h"
|
||||
@@ -916,9 +915,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
|
||||
std::vector<std::array<std::string, 9>> table_data;
|
||||
table_data.reserve(devices.size());
|
||||
|
||||
// same data as the table below, for --log-jsonl consumers
|
||||
common_json rows = common_json::array();
|
||||
const std::string template_header = "%s: | %s | %s %s %s %s %s %s %s |\n";
|
||||
const std::string template_gpu = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
|
||||
const std::string template_other = "%s: | %s | %s %s %s = %s + %s + %s %s |\n";
|
||||
@@ -993,19 +989,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
std::to_string(unaccounted / static_cast<int64_t>(MiB))});
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "device"},
|
||||
{"name", name},
|
||||
{"description", desc},
|
||||
{"total", total / MiB},
|
||||
{"free", free / MiB},
|
||||
{"self", self / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
{"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
|
||||
});
|
||||
}
|
||||
|
||||
// print memory breakdown for host:
|
||||
@@ -1021,15 +1004,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
std::to_string(mb_host.context / MiB),
|
||||
std::to_string(mb_host.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "host"},
|
||||
{"name", "Host"},
|
||||
{"self", self / MiB},
|
||||
{"model", mb_host.model / MiB},
|
||||
{"context", mb_host.context / MiB},
|
||||
{"compute", mb_host.compute / MiB},
|
||||
});
|
||||
}
|
||||
|
||||
// print memory breakdown for all remaining buffer types:
|
||||
@@ -1051,16 +1025,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
std::to_string(mb.context / MiB),
|
||||
std::to_string(mb.compute / MiB),
|
||||
""}); // unaccounted
|
||||
|
||||
rows.push_back({
|
||||
{"kind", "buffer_type"},
|
||||
{"name", name},
|
||||
{"self", self / MiB},
|
||||
{"model", mb.model / MiB},
|
||||
{"context", mb.context / MiB},
|
||||
{"compute", mb.compute / MiB},
|
||||
});
|
||||
|
||||
seen_buffer_types.insert(buft);
|
||||
}
|
||||
|
||||
@@ -1078,11 +1042,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
|
||||
__func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),
|
||||
td[6].c_str(), td[7].c_str(), td[8].c_str());
|
||||
}
|
||||
|
||||
LOG_JSON("fit_memory_breakdown", common_json({
|
||||
{"unit", "MiB"},
|
||||
{"rows", rows},
|
||||
}));
|
||||
}
|
||||
|
||||
void common_fit_print(
|
||||
|
||||
+42
-31
@@ -44,7 +44,8 @@ static fs::path get_cache_directory() {
|
||||
{HOME_DIR, fs::path(".cache") / "huggingface" / "hub"}
|
||||
};
|
||||
for (const auto & entry : entries) {
|
||||
if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) {
|
||||
if (auto * p = std::getenv(entry.var); p && *p) {
|
||||
fs::path base(p);
|
||||
return entry.path.empty() ? base : base / entry.path;
|
||||
}
|
||||
}
|
||||
@@ -61,10 +62,6 @@ static fs::path get_cache_directory() {
|
||||
return cache;
|
||||
}
|
||||
|
||||
std::string get_cache_path() {
|
||||
return fs_path_to_utf8(get_cache_directory());
|
||||
}
|
||||
|
||||
static std::string folder_name_to_repo(const std::string & folder) {
|
||||
constexpr std::string_view prefix = "models--";
|
||||
if (folder.rfind(prefix, 0)) {
|
||||
@@ -172,6 +169,28 @@ static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) {
|
||||
return b_end == b.end();
|
||||
}
|
||||
|
||||
static void safe_write_file(const fs::path & path, const std::string & data) {
|
||||
fs::path path_tmp = path.string() + ".tmp";
|
||||
|
||||
if (path.has_parent_path()) {
|
||||
fs::create_directories(path.parent_path());
|
||||
}
|
||||
|
||||
std::ofstream file(path_tmp);
|
||||
file << data;
|
||||
file.close();
|
||||
|
||||
std::error_code ec;
|
||||
|
||||
if (!file.fail()) {
|
||||
fs::rename(path_tmp, path, ec);
|
||||
}
|
||||
if (file.fail() || ec) {
|
||||
fs::remove(path_tmp, ec);
|
||||
throw std::runtime_error("failed to write file: " + path.string());
|
||||
}
|
||||
}
|
||||
|
||||
static common_json api_get(const std::string & url,
|
||||
const std::string & token) {
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
@@ -218,7 +237,6 @@ static std::string get_repo_commit(const std::string & repo_id,
|
||||
fs::path refs_path = get_repo_path(repo_id) / "refs";
|
||||
std::string name;
|
||||
std::string commit;
|
||||
fs::path name_path;
|
||||
|
||||
for (const auto & branch : json["branches"]) {
|
||||
if (!branch.is_object() ||
|
||||
@@ -229,28 +247,24 @@ static std::string get_repo_commit(const std::string & repo_id,
|
||||
std::string _name = branch["name"].get<std::string>();
|
||||
std::string _commit = branch["targetCommit"].get<std::string>();
|
||||
|
||||
if (!is_valid_commit(_commit)) {
|
||||
LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
|
||||
if (!is_valid_subpath(refs_path, _name)) {
|
||||
LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
|
||||
continue;
|
||||
}
|
||||
const fs::path candidate = fs::u8path(_name);
|
||||
|
||||
if (!is_valid_subpath(refs_path, candidate)) {
|
||||
LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
|
||||
if (!is_valid_commit(_commit)) {
|
||||
LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_name == "main") {
|
||||
name = _name;
|
||||
commit = _commit;
|
||||
name_path = candidate;
|
||||
break;
|
||||
}
|
||||
|
||||
if (name.empty() || commit.empty()) {
|
||||
name = _name;
|
||||
commit = _commit;
|
||||
name_path = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,7 +273,7 @@ static std::string get_repo_commit(const std::string & repo_id,
|
||||
return {};
|
||||
}
|
||||
|
||||
fs_write_atomic(refs_path / name_path, commit);
|
||||
safe_write_file(refs_path / name, commit);
|
||||
return commit;
|
||||
|
||||
} catch (const common_json_error & e) {
|
||||
@@ -308,9 +322,7 @@ hf_files get_repo_files(const std::string & repo_id,
|
||||
file.repo_id = repo_id;
|
||||
file.path = item["path"].get<std::string>();
|
||||
|
||||
const fs::path subpath = fs::u8path(file.path);
|
||||
|
||||
if (!is_valid_subpath(commit_path, subpath)) {
|
||||
if (!is_valid_subpath(commit_path, file.path)) {
|
||||
LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str());
|
||||
continue;
|
||||
}
|
||||
@@ -330,12 +342,12 @@ hf_files get_repo_files(const std::string & repo_id,
|
||||
|
||||
file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
|
||||
|
||||
fs::path final_path = commit_path / subpath;
|
||||
file.final_path = fs_path_to_utf8(final_path);
|
||||
fs::path final_path = commit_path / file.path;
|
||||
file.final_path = final_path.string();
|
||||
|
||||
if (!file.oid.empty() && !fs::exists(final_path)) {
|
||||
fs::path local_path = blobs_path / file.oid;
|
||||
file.local_path = fs_path_to_utf8(local_path);
|
||||
file.local_path = local_path.string();
|
||||
} else {
|
||||
file.local_path = file.final_path;
|
||||
}
|
||||
@@ -381,8 +393,8 @@ static std::string get_cached_ref(const fs::path & repo_path) {
|
||||
}
|
||||
|
||||
hf_files get_cached_files(const std::string & repo_id) {
|
||||
const fs::path cache_path = get_cache_directory();
|
||||
if (!fs::exists(cache_path)) {
|
||||
fs::path cache_dir = get_cache_directory();
|
||||
if (!fs::exists(cache_dir)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -393,7 +405,7 @@ hf_files get_cached_files(const std::string & repo_id) {
|
||||
|
||||
hf_files files;
|
||||
|
||||
for (const auto & repo : fs::directory_iterator(cache_path)) {
|
||||
for (const auto & repo : fs::directory_iterator(cache_dir)) {
|
||||
if (!repo.is_directory()) {
|
||||
continue;
|
||||
}
|
||||
@@ -402,7 +414,7 @@ hf_files get_cached_files(const std::string & repo_id) {
|
||||
if (!fs::exists(snapshots_path)) {
|
||||
continue;
|
||||
}
|
||||
std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
|
||||
std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
|
||||
|
||||
if (!is_valid_repo_id(_repo_id)) {
|
||||
continue;
|
||||
@@ -425,9 +437,8 @@ hf_files get_cached_files(const std::string & repo_id) {
|
||||
if (!path.empty()) {
|
||||
hf_file file;
|
||||
file.repo_id = _repo_id;
|
||||
const auto generic_path = path.generic_u8string();
|
||||
file.path = std::string(generic_path.begin(), generic_path.end());
|
||||
file.local_path = fs_path_to_utf8(entry.path());
|
||||
file.path = path.generic_string();
|
||||
file.local_path = entry.path().string();
|
||||
file.final_path = file.local_path;
|
||||
files.push_back(std::move(file));
|
||||
}
|
||||
@@ -441,8 +452,8 @@ std::string finalize_file(const hf_file & file) {
|
||||
static std::atomic<bool> symlinks_disabled{false};
|
||||
|
||||
std::error_code ec;
|
||||
fs::path local_path = fs::u8path(file.local_path);
|
||||
fs::path final_path = fs::u8path(file.final_path);
|
||||
fs::path local_path(file.local_path);
|
||||
fs::path final_path(file.final_path);
|
||||
|
||||
if (local_path == final_path || fs::exists(final_path, ec)) {
|
||||
return file.final_path;
|
||||
@@ -489,7 +500,7 @@ bool remove_cached_repo(const std::string & repo_id) {
|
||||
std::error_code ec;
|
||||
auto removed = fs::remove_all(repo_path, ec);
|
||||
if (ec) {
|
||||
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
|
||||
LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
|
||||
return false;
|
||||
}
|
||||
return removed > 0;
|
||||
|
||||
@@ -32,7 +32,4 @@ std::string finalize_file(const hf_file & file);
|
||||
// Remove the entire cached directory for a repo, returns true if removed
|
||||
bool remove_cached_repo(const std::string & repo_id);
|
||||
|
||||
// Returns the HuggingFace hub cache path
|
||||
std::string get_cache_path();
|
||||
|
||||
} // namespace hf_cache
|
||||
|
||||
+12
-29
@@ -98,10 +98,9 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int64_t datasets_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);
|
||||
const int64_t datasets_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);
|
||||
const int64_t chunk_count_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);
|
||||
const int64_t chunk_size_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);
|
||||
const int64_t nextn_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_N_LAYER_NEXTN);
|
||||
|
||||
if (datasets_key != -1 && gguf_get_kv_type(ctx_gguf, datasets_key) == GGUF_TYPE_ARRAY &&
|
||||
gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) {
|
||||
@@ -112,42 +111,33 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
|
||||
}
|
||||
}
|
||||
|
||||
imatrix.has_metadata = datasets_key != -1 && chunk_count_key != -1 && chunk_size_key != -1;
|
||||
imatrix.chunk_count = chunk_count_key != -1 ? gguf_get_val_u32(ctx_gguf, chunk_count_key) : 0;
|
||||
imatrix.chunk_size = chunk_size_key != -1 ? gguf_get_val_u32(ctx_gguf, chunk_size_key) : 0;
|
||||
imatrix.n_layer_nextn = nextn_key != -1 ? gguf_get_val_u32(ctx_gguf, nextn_key) : 0;
|
||||
imatrix.has_metadata = (datasets_key != -1 && chunk_count_key != -1 && chunk_size_key != -1);
|
||||
imatrix.chunk_count = (chunk_count_key != -1) ? gguf_get_val_u32(ctx_gguf, chunk_count_key) : 0;
|
||||
imatrix.chunk_size = (chunk_size_key != -1) ? gguf_get_val_u32(ctx_gguf, chunk_size_key) : 0;
|
||||
|
||||
const std::string in_sum_suffix{ ".in_sum" };
|
||||
const std::string in_sum2_suffix{ ".in_sum2" };
|
||||
const std::string counts_suffix{ ".counts" };
|
||||
|
||||
struct sum_tensors {
|
||||
struct ggml_tensor * in_sum = nullptr;
|
||||
struct ggml_tensor * in_sum2 = nullptr;
|
||||
struct ggml_tensor * counts = nullptr;
|
||||
};
|
||||
std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;
|
||||
|
||||
std::map<std::string, sum_tensors> sums_counts_for;
|
||||
for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
|
||||
std::string name = cur->name;
|
||||
|
||||
if (name.empty()) { continue; }
|
||||
|
||||
if (string_remove_suffix(name, in_sum_suffix)) {
|
||||
sums_counts_for[std::move(name)].in_sum = cur;
|
||||
} else if (string_remove_suffix(name, in_sum2_suffix)) {
|
||||
sums_counts_for[std::move(name)].in_sum2 = cur;
|
||||
if (string_remove_suffix(name, in_sum2_suffix)) {
|
||||
sums_counts_for[std::move(name)].first = cur;
|
||||
} else if (string_remove_suffix(name, counts_suffix)) {
|
||||
sums_counts_for[std::move(name)].counts = cur;
|
||||
sums_counts_for[std::move(name)].second = cur;
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & sc : sums_counts_for) {
|
||||
const std::string & name = sc.first;
|
||||
const struct ggml_tensor * in_sum = sc.second.in_sum;
|
||||
const struct ggml_tensor * in_sum2 = sc.second.in_sum2;
|
||||
const struct ggml_tensor * counts = sc.second.counts;
|
||||
const struct ggml_tensor * in_sum2 = sc.second.first;
|
||||
const struct ggml_tensor * counts = sc.second.second;
|
||||
|
||||
if (!in_sum2 || !counts || (in_sum != nullptr && ggml_nelements(in_sum) != ggml_nelements(in_sum2))) {
|
||||
if (!in_sum2 || !counts) {
|
||||
LOG_ERR("%s: mismatched sums and counts for %s\n", __func__, name.c_str());
|
||||
gguf_free(ctx_gguf);
|
||||
ggml_free(ctx);
|
||||
@@ -175,13 +165,6 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) {
|
||||
for (int64_t j = 0; j < ncounts; ++j) {
|
||||
e.counts[j] = std::lround(((const float *) counts->data)[j]);
|
||||
}
|
||||
|
||||
if (in_sum && ggml_nelements(in_sum) == nval) {
|
||||
e.activations.resize(nval);
|
||||
for (int64_t j = 0; j < nval; ++j) {
|
||||
e.activations[j] = ((const float *) in_sum->data)[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
gguf_free(ctx_gguf);
|
||||
|
||||
@@ -8,12 +8,9 @@
|
||||
inline constexpr const char * LLM_KV_IMATRIX_DATASETS = "imatrix.datasets";
|
||||
inline constexpr const char * LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";
|
||||
inline constexpr const char * LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size";
|
||||
inline constexpr const char * LLM_KV_IMATRIX_STATS_SCHEMA = "imatrix.stats_schema";
|
||||
inline constexpr const char * LLM_KV_IMATRIX_N_LAYER_NEXTN = "imatrix.n_layer_nextn";
|
||||
|
||||
struct common_imatrix_entry {
|
||||
std::vector<float> sums;
|
||||
std::vector<float> activations;
|
||||
std::vector<int64_t> counts;
|
||||
};
|
||||
|
||||
@@ -22,7 +19,6 @@ struct common_imatrix {
|
||||
std::vector<std::string> datasets;
|
||||
int32_t chunk_count = 0;
|
||||
int32_t chunk_size = 0;
|
||||
int32_t n_layer_nextn = 0;
|
||||
bool is_legacy = false;
|
||||
bool has_metadata = false;
|
||||
};
|
||||
|
||||
@@ -117,7 +117,6 @@ caps caps_get(jinja::program & prog) {
|
||||
|
||||
JJ_DEBUG("%s\n", ">>> Running capability check: typed content");
|
||||
|
||||
bool checks_for_string = false;
|
||||
static const std::string content_marker = "STRING_MARKER";
|
||||
|
||||
// case: typed content support
|
||||
@@ -137,10 +136,6 @@ caps caps_get(jinja::program & prog) {
|
||||
[&](context &, bool success, value & messages, value &, const std::string & rendered) {
|
||||
auto & content = messages->at(0)->at("content");
|
||||
caps_print_stats(content, "messages[0].content");
|
||||
if (has_op(content, "test_is_string")) {
|
||||
// checked if content is string
|
||||
checks_for_string = true;
|
||||
}
|
||||
bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
|
||||
if (used_as_array) {
|
||||
// accessed as an array
|
||||
@@ -156,33 +151,6 @@ caps caps_get(jinja::program & prog) {
|
||||
}
|
||||
);
|
||||
|
||||
if (checks_for_string) {
|
||||
caps_try_execute(
|
||||
prog,
|
||||
[&]() {
|
||||
// messages
|
||||
return json::array({
|
||||
{
|
||||
{"role", "user"},
|
||||
{"content", json::array({
|
||||
})}
|
||||
}
|
||||
});
|
||||
},
|
||||
nullptr, // ctx_fn
|
||||
nullptr, // tools_fn
|
||||
[&](context &, bool success, value & messages, value &, const std::string &) {
|
||||
auto & content = messages->at(0)->at("content");
|
||||
caps_print_stats(content, "messages[0].content");
|
||||
bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
|
||||
if (used_as_array && success) {
|
||||
// accessed as an array
|
||||
result.supports_typed_content = true;
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
JJ_DEBUG("%s\n", ">>> Running capability check: system prompt");
|
||||
|
||||
// case: system prompt support
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user