quantize: row-slab stream to avoid thread starvation

.devops/intel.Dockerfile

@@ -1,4 +1,4 @@
-ARG ONEAPI_VERSION=2026.1.1-devel-ubuntu24.04
+ARG ONEAPI_VERSION=2025.3.3-0-devel-ubuntu24.04
 ARG BUILD_DATE=N/A
 ARG APP_VERSION=N/A
 ARG APP_REVISION=N/A
@@ -19,7 +19,7 @@ RUN npm ci
 COPY tools/ui/ ./
 RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
 
-FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS build
+FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
 
 ARG GGML_SYCL_F16=ON
 ARG LEVEL_ZERO_VERSION=1.28.2
@@ -59,7 +59,7 @@ RUN mkdir -p /app/full \
     && cp requirements.txt /app/full \
     && cp .devops/tools.sh /app/full/tools.sh
 
-FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base
+FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base
 
 ARG BUILD_DATE=N/A
 ARG APP_VERSION=N/A

.devops/musa.Dockerfile

@@ -1,9 +1,10 @@
 ARG UBUNTU_VERSION=22.04
 # This needs to generally match the container host's environment.
+ARG MUSA_VERSION=rc4.3.0
 # Target the MUSA build image
-ARG BASE_MUSA_DEV_CONTAINER=registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu${UBUNTU_VERSION}-s5000
+ARG BASE_MUSA_DEV_CONTAINER=docker.io/mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64
 
-ARG BASE_MUSA_RUN_CONTAINER=registry.mthreads.com/mcconline/musa_sdk:5.2.0-runtime-ubuntu${UBUNTU_VERSION}-s5000
+ARG BASE_MUSA_RUN_CONTAINER=docker.io/mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64
 
 ARG BUILD_DATE=N/A
 ARG APP_VERSION=N/A
@@ -36,10 +37,7 @@ RUN apt-get update && \
     python3-pip \
     git \
     libssl-dev \
-    libgomp1 \
-    musa-mualg-5-2 \
-    musa-muthrust-5-2 \
-    libmthreads-compute
+    libgomp1
 
 WORKDIR /app
 
@@ -82,16 +80,13 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
       org.opencontainers.image.source=$IMAGE_SOURCE
 
 RUN apt-get update \
-    && apt-get install -y libgomp1 curl ffmpeg libmthreads-compute \
+    && apt-get install -y libgomp1 curl ffmpeg \
     && apt autoremove -y \
     && apt clean -y \
     && rm -rf /tmp/* /var/tmp/* \
     && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \
     && find /var/cache -type f -delete
 
-# The MUSA runtime image does not register its library directory
-RUN echo "/usr/local/musa/lib" > /etc/ld.so.conf.d/musa-runtime.conf && ldconfig
-
 COPY --from=build /app/lib/ /app
 
 ### Full

.devops/nix/package.nix

@@ -31,7 +31,7 @@
     ]
     && blas.meta.available,
   useCuda ? config.cudaSupport,
-  useMetalKit ? stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isDarwin,
+  useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin,
   # Increases the runtime closure size by ~700M
   useMpi ? false,
   useRocm ? config.rocmSupport,
@@ -92,7 +92,7 @@ let
 
   cudaBuildInputs = with cudaPackages; [
     cuda_cudart
-    cccl # <nv/target>
+    cuda_cccl # <nv/target>
     libcublas
   ];
 
@@ -166,7 +166,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
   # `xcrun` is used find the path of the Metal compiler, which is varible
   # and not on $PATH
   # see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
-  __noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
+  __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
 
   nativeBuildInputs =
     [
@@ -181,10 +181,10 @@ effectiveStdenv.mkDerivation (finalAttrs: {
       autoAddDriverRunpath
     ]
     ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
-    ++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
+    ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
 
   buildInputs =
-    optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
+    optionals effectiveStdenv.isDarwin darwinBuildInputs
     ++ optionals useCuda cudaBuildInputs
     ++ optionals useMpi [ mpi ]
     ++ optionals useRocm rocmBuildInputs
@@ -245,7 +245,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
 
     # Configurations that are known to result in build failures. Can be
     # overridden by importing Nixpkgs with `allowBroken = true`.
-    broken = (useMetalKit && !effectiveStdenv.hostPlatform.isDarwin);
+    broken = (useMetalKit && !effectiveStdenv.isDarwin);
 
     description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
     homepage = "https://github.com/ggml-org/llama.cpp/";

.devops/openvino.Dockerfile

@@ -1,18 +1,18 @@
-ARG OPENVINO_VERSION_MAJOR=2026.4
-ARG OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3
+ARG OPENVINO_VERSION_MAJOR=2026.3
+ARG OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c
 ARG UBUNTU_VERSION=24.04
 
 # Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
-ARG IGC_VERSION=v2.40.13
-ARG IGC_VERSION_FULL=2_2.40.13+22418
-ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
-ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
+ARG IGC_VERSION=v2.38.2
+ARG IGC_VERSION_FULL=2_2.38.2+22051
+ARG COMPUTE_RUNTIME_VERSION=26.27.39122.11
+ARG COMPUTE_RUNTIME_VERSION_FULL=26.27.39122.11-0
 ARG IGDGMM_VERSION=22.10.0
 
 # Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
-ARG NPU_DRIVER_VERSION=v1.38.0
-ARG NPU_DRIVER_FULL=v1.38.0.20260910-34487311128
-ARG LIBZE1_VERSION=1.32.0-1~24.04~ppa1
+ARG NPU_DRIVER_VERSION=v1.35.0
+ARG NPU_DRIVER_FULL=v1.35.0.20260722-29947505341
+ARG LIBZE1_VERSION=1.28.2-1~24.04~ppa1
 
 # Optional proxy build arguments
 ARG http_proxy=
@@ -173,7 +173,7 @@ RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \
     fi; \
     DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \
     if [ ! -f "$DEB" ]; then \
-        wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260830T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
+        wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260606T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
     fi; \
     mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \
     apt-get update; \

.ecrc

@@ -1,5 +1,5 @@
 {
-  "Exclude": ["^\\.gitmodules$", "stb_image\\.h", "examples/test-cmake/build/", "examples/test-cmake/build-subdir/"],
+  "Exclude": ["^\\.gitmodules$", "stb_image\\.h"],
   "Disable": {
     "IndentSize": true
   }

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
-blank_issues_enabled: false
+blank_issues_enabled: true
 contact_links:
   - name: Got an idea?
     url: https://github.com/ggml-org/llama.cpp/discussions/categories/ideas

.github/actions/get-tag-name/action.yml

@@ -14,7 +14,7 @@ runs:
       run: |
         BUILD_NUMBER="$(git rev-list --count HEAD)"
         SHORT_HASH="$(git rev-parse --short=7 HEAD)"
-        if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
+        if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
           echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
         else
           SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')

.github/actions/windows-setup-cuda/action.yml

@@ -100,36 +100,36 @@ runs:
           echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
           echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
 
-    - name: Install Cuda Toolkit 13.4 for x64
-      if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }}
+    - name: Install Cuda Toolkit 13.3
+      if: ${{ inputs.cuda_version == '13.3' }}
       shell: pwsh
       run: |
-          mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
+          mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3"
           choco install unzip -y
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
-          unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
-          echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip"
+          unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3"
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y
+          echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
 
     - name: Install Cuda Toolkit 13.4 for ARM64
       if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }}
@@ -137,19 +137,19 @@ runs:
       run: |
           mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
           choco install unzip -y
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cccl-windows-x86_64-13.3.4.1.2-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_crt-windows-x86_64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_nvcc-windows-x86_64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/libnvvm-windows-x86_64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_cudart-windows-arm64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/libcublas-windows-arm64-13.7.0.10-archive.zip"
           unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.1.2-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.10-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
           echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
           echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
           echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8

.github/actions/windows-setup-rocm/action.yml

@@ -24,7 +24,7 @@ runs:
 
         write-host "Installing ROCm wheels for multi-arch support"
         # Install ROCm wheels for multi-arch support (this may take several minutes)
-        python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ inputs.version }}"
+        python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ inputs.version }}"
 
         # Pre-expand the devel tree so it is included in the cache
         write-host "Initializing ROCm devel tree"

.github/workflows/build-and-test-snapdragon.yml

@@ -29,7 +29,7 @@ concurrency:
 
 jobs:
   android-ndk-snapdragon:
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
     container:
       image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
     defaults:
@@ -59,7 +59,7 @@ jobs:
           path: pkg-snapdragon/llama.cpp
 
   linux-iot-snapdragon:
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
     container:
       image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
     defaults:

.github/workflows/build-android.yml

@@ -33,7 +33,7 @@ env:
 
 jobs:
   default:
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
 
     steps:
       - name: Clone
@@ -49,7 +49,7 @@ jobs:
           distribution: zulu
 
       - name: Setup Android SDK
-        uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
+        uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
         with:
           log-accepted-android-sdk-licenses: false
 
@@ -59,7 +59,7 @@ jobs:
           ./gradlew build --no-daemon
 
   ndk:
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
     container:
       image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3'
     defaults:
@@ -93,7 +93,7 @@ jobs:
           path: pkg-adb/llama.cpp
 
   arm64:
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
 
     env:
       NDK_VERSION: "29.0.14206865"
@@ -110,7 +110,7 @@ jobs:
       #   cache on:  https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
       #
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.21
       #  with:
       #    key: android-ubuntu-arm64
       #    evict-old-files: 1d
@@ -123,7 +123,7 @@ jobs:
           distribution: temurin
 
       - name: Setup Android SDK
-        uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
+        uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
         with:
           log-accepted-android-sdk-licenses: false
 

.github/workflows/build-apple.yml

@@ -33,7 +33,6 @@ concurrency:
 env:
   GGML_NLOOP: 3
   GGML_N_THREADS: 1
-  GGML_SCHED_DEBUG_REALLOC: 1
   LLAMA_ARG_LOG_COLORS: 1
   LLAMA_ARG_LOG_PREFIX: 1
   LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -48,19 +47,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: apple-arm64
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: apple-arm64
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -75,32 +66,23 @@ jobs:
             -DGGML_RPC=ON \
             -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
           time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: apple-arm64
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      - name: Check for leaks
-        run: |
-          cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1)
-          leaks -atExit -- "${cmd[@]}"
-          # Graphics devices are leaked by Metal in Apple code sometimes, so we ignore those leaks
-          OBJC_DEBUG_MISSING_POOLS=YES "${cmd[@]}" 2>&1 | awk '{ print } index($0, "autoreleased with no pool in place") && !/class [a-zA-Z0-9]+Device autoreleased/ { found = 1 } END { exit found }'
+          leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1
 
       - name: Test
         id: cmake_test
         run: |
           cd build
-          # ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
-          ctest -L main -E "test-llama-archs|test-save-load-state" --verbose --timeout 900
+          ctest -L main -E "test-llama-archs" --verbose --timeout 900
+
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: apple-arm64
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
 
   macos-latest-x64:
     runs-on: macos-15-intel
@@ -111,19 +93,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: apple-x64
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: apple-x64
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -140,24 +114,22 @@ jobs:
             -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
           time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: apple-x64
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
           cd build
           ctest -L main --verbose --timeout 900
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: apple-x64
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+
   macos-latest-ios-xcode:
     runs-on: macos-latest
 

.github/workflows/build-cache.yml

@@ -41,8 +41,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.4"
-      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone
@@ -69,8 +69,8 @@ jobs:
 
     env:
       # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.4"
-      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone

.github/workflows/build-cmake-pkg.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   linux:
-    runs-on: [self-hosted, Linux, CPU]
+    runs-on: [self-hosted, Linux]
     steps:
       - uses: actions/checkout@v6
         with:

.github/workflows/build-cpu.yml

@@ -37,7 +37,6 @@ concurrency:
 env:
   GGML_NLOOP: 3
   GGML_N_THREADS: 1
-  GGML_SCHED_DEBUG_REALLOC: 1
   LLAMA_ARG_LOG_COLORS: 1
   LLAMA_ARG_LOG_PREFIX: 1
   LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -63,10 +62,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cpu-${{ matrix.os }}
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build Dependencies
         id: build_depends
@@ -89,16 +89,7 @@ jobs:
         run: |
           export PIP_BREAK_SYSTEM_PACKAGES="1"
           python3 -m pip install --upgrade pip setuptools
-          pip3 install ./gguf-py jinja2==3.1.6
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: cpu-${{ matrix.os }}
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          pip3 install ./gguf-py
 
       - name: Build
         id: cmake_build
@@ -109,23 +100,11 @@ jobs:
             -DGGML_RPC=ON
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: cpu-${{ matrix.os }}
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
           cd build
-          ctest -L 'main|python' --verbose --timeout 900
+          ctest -L main --verbose --timeout 900
 
       - name: Test llama2c conversion
         id: llama2c_test
@@ -138,6 +117,18 @@ jobs:
           ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
           ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
 
+      # note: real deletion only on push to master (same condition as the ccache save),
+      #       dry-run otherwise (the token is read-only on PRs from forks)
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: cpu-${{ matrix.os }}
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+
   windows:
     name: windows / ${{ matrix.build }}
     runs-on: windows-2025
@@ -165,7 +156,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cpu-windows-2025-${{ matrix.build }}
           variant: ccache
@@ -222,6 +213,7 @@ jobs:
       #     7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
       #     $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
       #     cd build
+      #     $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
       #     & $sde -future -- ctest -L main -C Release --verbose --timeout 900
 
       - name: ccache-clear

.github/workflows/build-cuda-ubuntu.yml

@@ -53,7 +53,7 @@ jobs:
           apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-24.04-cuda
           save: false
@@ -61,14 +61,14 @@ jobs:
       - name: ccache-buckets-restore
         uses: ./.github/actions/ccache-buckets
         env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
         with:
           key: cuda-ubuntu-24.04-cuda
           folder: llama.cpp
           hf_bucket: ggml-org/cache
 
       - name: Build with CMake
-        # TODO: Drop GGML_CUDA_CCCL_VERSION when this job uses CTK >= 13.5, which bundles CCCL >= 3.5.
+        # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
         run: |
           cmake -S . -B build -G Ninja \
             -DLLAMA_FATAL_WARNINGS=ON \
@@ -77,7 +77,7 @@ jobs:
             -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \
             -DGGML_NATIVE=OFF \
             -DGGML_CUDA=ON \
-            -DGGML_CUDA_CCCL_VERSION=v3.4.3
+            -DGGML_CUDA_CUB_3DOT2=ON
           cmake --build build
 
       - name: ccache-buckets-save
@@ -108,7 +108,7 @@ jobs:
           sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-22.04-hip
           save: false
@@ -116,7 +116,7 @@ jobs:
       - name: ccache-buckets-restore
         uses: ./.github/actions/ccache-buckets
         env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
         with:
           key: cuda-ubuntu-22.04-hip
           folder: llama.cpp
@@ -145,7 +145,7 @@ jobs:
 
   musa:
     runs-on: ubuntu-22.04
-    container: registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu22.04-s5000
+    container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
 
     steps:
       - name: Clone
@@ -156,10 +156,10 @@ jobs:
         id: depends
         run: |
           apt-get update
-          apt-get install -y build-essential git cmake libssl-dev jq python3-venv musa-mualg-5-2 musa-muthrust-5-2 libmthreads-compute
+          apt-get install -y build-essential git cmake libssl-dev jq
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-22.04-musa
           save: false
@@ -167,7 +167,7 @@ jobs:
       - name: ccache-buckets-restore
         uses: ./.github/actions/ccache-buckets
         env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
         with:
           key: cuda-ubuntu-22.04-musa
           folder: llama.cpp
@@ -177,9 +177,8 @@ jobs:
         id: cmake_build
         run: |
           cmake -B build -S . \
-            -DGGML_MUSA=ON \
-            -DMUSA_ARCHITECTURES=31
-          cmake --build build --config Release -j $(nproc)
+            -DGGML_MUSA=ON
+          time cmake --build build --config Release -j $(nproc)
 
       - name: ccache-buckets-save
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}

.github/workflows/build-cuda-windows.yml

@@ -31,16 +31,15 @@ jobs:
     strategy:
       matrix:
         include:
-          # CTK >= 13.5 bundles CCCL >= 3.5; omit GGML_CUDA_CCCL_VERSION for those versions.
           - cuda: '12.4'
             arch: x64
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
-          - cuda: '13.4'
+            defines: '-DGGML_CUDA_CUB_3DOT2=ON'
+          - cuda: '13.3'
             arch: x64
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
+            defines: ''
           - cuda: '13.4'
             arch: arm64
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
+            defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
 
     steps:
       - name: Clone
@@ -48,7 +47,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
 
@@ -153,7 +152,7 @@ jobs:
           & "${env:HIP_PATH}\lib\llvm\bin\clang.exe" --version
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           # TODO: this build does not match the build in release.yml, so we use a different cache key
           #       ideally, the builds should match, similar to the CUDA build above so that we would be able

.github/workflows/build-ibm.yml

@@ -34,14 +34,9 @@ env:
   LLAMA_ARG_LOG_TIMESTAMPS: 1
 
 jobs:
-  ubuntu-24-s390x:
-    name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
-    runs-on: ubuntu-24.04-s390x
 
-    strategy:
-      fail-fast: false
-      matrix:
-        vxe: ["ON", "OFF"]  # `-DGGML_VXE=ON/OFF`
+  ubuntu-24-s390x:
+    runs-on: ubuntu-24.04-s390x
 
     steps:
       - name: Clone
@@ -82,8 +77,7 @@ jobs:
         run: |
           cmake -B build \
             -DLLAMA_FATAL_WARNINGS=ON \
-            -DGGML_RPC=ON \
-            -DGGML_VXE=${{ matrix.vxe }}
+            -DGGML_RPC=ON
           time cmake --build build --config Release -j $(nproc)
 
       - name: Test
@@ -100,46 +94,6 @@ jobs:
           wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
           ./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
 
-  ubuntu-26-zdnn-s390x:
-    name: ubuntu-26-zdnn-s390x
-    runs-on: ubuntu-24.04-s390x
-    container: ubuntu:26.04  # required to get GCC 15.1 and binutils 2.44
-    defaults:
-      run:
-        shell: bash
-
-    steps:
-      - name: Build Dependencies
-        id: build_depends
-        run: |
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential cmake git ca-certificates \
-            libssl-dev libzdnn-dev
-
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Toolchain workaround (GCC 15)
-        run: |
-          apt-get install -y gcc-15 g++-15
-          echo "CC=gcc-15" >> "$GITHUB_ENV"
-          echo "CXX=g++-15" >> "$GITHUB_ENV"
-
-      - name: Build with zDNN Backend
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DGGML_NATIVE=OFF \
-            -DGGML_VXE=ON \
-            -DGGML_ZDNN=ON \
-            -DGGML_RPC=ON \
-            -DCMAKE_C_FLAGS="-march=arch15" \
-            -DCMAKE_CXX_FLAGS="-march=arch15"
-          time cmake --build build --config Release -j $(nproc)
-
   ubuntu-24-ppc64le:
     runs-on: ubuntu-24.04-ppc64le
 

.github/workflows/build-msys.yml

@@ -35,7 +35,7 @@ jobs:
         uses: actions/checkout@v6
 
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.16
       #  with:
       #    key: msys-windows-2025-x64
       #    variant: ccache

.github/workflows/build-opencl.yml

@@ -44,7 +44,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: opencl-windows-2025-x64
           variant: ccache

.github/workflows/build-openvino.yml

@@ -32,8 +32,6 @@ env:
   LLAMA_ARG_LOG_COLORS: 1
   LLAMA_ARG_LOG_PREFIX: 1
   LLAMA_ARG_LOG_TIMESTAMPS: 1
-  # TODO: fix failing tests on OpenVINO backend
-  CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state"
 
 jobs:
   ubuntu-24-openvino:
@@ -41,8 +39,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.4"
-      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone
@@ -80,24 +78,26 @@ jobs:
 
       - name: Test (CPU)
         id: cmake_test_cpu
+        # TODO: fix and re-enable the `test-llama-archs` test below
         run: |
           cd ${{ github.workspace }}
-          ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
+          ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 2000
 
       - name: Test (GPU)
         id: cmake_test_gpu
+        # TODO: fix and re-enable the `test-llama-archs` test below
         run: |
           cd ${{ github.workspace }}
           export GGML_OPENVINO_DEVICE=GPU
-          ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
+          ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 3000
 
   openvino-windows-2022:
     runs-on: windows-2022
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.4"
-      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone
@@ -105,7 +105,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: openvino-windows-2022
           variant: ccache
@@ -159,13 +159,14 @@ jobs:
       - name: Test (CPU)
         id: cmake_test_cpu
         shell: cmd
+        # TODO: fix and re-enable the `test-llama-archs` test below
         run: |
           REM Find extracted OpenVINO folder dynamically
           for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
           call "%OPENVINO_ROOT%\setupvars.bat"
 
           cd build
-          ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
+          ctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000
 
       - name: ccache-clear
         uses: ./.github/actions/ccache-clear

.github/workflows/build-riscv.yml

@@ -67,7 +67,7 @@ jobs:
 
       # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
       #  with:
       #    key: riscv-ubuntu-native
       #    evict-old-files: 1d
@@ -106,3 +106,83 @@ jobs:
           wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
           ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
           ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
+
+  ubuntu-riscv64-native-sanitizer:
+    runs-on: ubuntu-24.04-riscv
+
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        sanitizer: [ADDRESS, THREAD, UNDEFINED]
+        build_type: [Debug]
+
+    steps:
+      - name: Install dependencies
+        run: |
+          # Set gcc-14 and g++-14 as the default compilers
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
+
+          git lfs install
+
+      - name: GCC version check
+        run: |
+          gcc --version
+          g++ --version
+
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
+      #- name: ccache
+      #  uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
+      #  with:
+      #    key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }}
+      #    evict-old-files: 1d
+      #    save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
+      - name: Build
+        id: cmake_build
+        if: ${{ matrix.sanitizer != 'THREAD' }}
+        run: |
+          cmake -B build \
+            -DLLAMA_OPENSSL=OFF \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DGGML_OPENMP=ON \
+            -DLLAMA_BUILD_EXAMPLES=ON \
+            -DLLAMA_BUILD_TOOLS=ON \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
+
+          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+
+      - name: Build (no OpenMP)
+        id: cmake_build_no_openmp
+        if: ${{ matrix.sanitizer == 'THREAD' }}
+        run: |
+          cmake -B build \
+            -DLLAMA_OPENSSL=OFF \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DGGML_OPENMP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=ON \
+            -DLLAMA_BUILD_TOOLS=ON \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
+
+          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+
+      - name: Test
+        id: cmake_test
+        run: |
+          cd build
+          ctest -L main --verbose --timeout 900

.github/workflows/build-sanitize.yml

@@ -55,7 +55,7 @@ jobs:
         uses: actions/checkout@v6
 
       # - name: ccache
-      #   uses: ggml-org/ccache-action@v1.2.24
+      #   uses: ggml-org/ccache-action@v1.2.21
       #   if: ${{ matrix.sanitizer != 'UNDEFINED' }}
       #   with:
       #     key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
@@ -101,6 +101,8 @@ jobs:
 
       - name: Test
         id: cmake_test
+        # skip run in Debug - very slow
+        if: ${{ matrix.sanitizer != 'UNDEFINED' }}
         run: |
           cd build
           ctest -L main -E tokenizer --verbose --timeout 900

.github/workflows/build-self-hosted.yml

@@ -0,0 +1,409 @@
+name: CI (self-hosted)
+
+on:
+  workflow_dispatch: # allows manual triggering
+  push:
+    branches:
+      - master
+    paths: [
+      '.github/workflows/build-self-hosted.yml',
+      '**/CMakeLists.txt',
+      '**/.cmake',
+      '**/*.h',
+      '**/*.hpp',
+      '**/*.c',
+      '**/*.cpp',
+      '**/*.cu',
+      '**/*.cuh',
+      '**/*.swift',
+      '**/*.m',
+      '**/*.metal',
+      '**/*.comp',
+      '**/*.glsl',
+      '**/*.wgsl'
+    ]
+
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths: [
+      '.github/workflows/build-self-hosted.yml',
+      '**/CMakeLists.txt',
+      '**/.cmake',
+      '**/*.h',
+      '**/*.hpp',
+      '**/*.c',
+      '**/*.cpp',
+      '**/*.cu',
+      '**/*.cuh',
+      '**/*.swift',
+      '**/*.m',
+      '**/*.metal',
+      '**/*.comp',
+      '**/*.glsl',
+      '**/*.wgsl'
+    ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
+  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+  GGML_NLOOP: 3
+  GGML_N_THREADS: 1
+  LLAMA_ARG_LOG_COLORS: 1
+  LLAMA_ARG_LOG_PREFIX: 1
+  LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+jobs:
+  gpu-cuda:
+    runs-on: [self-hosted, Linux, NVIDIA]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          nvidia-smi
+          GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-rocm:
+    runs-on: [self-hosted, Linux, AMD]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
+        # issue on integrated RDNA3.5 (gfx1151) where batched inference returns
+        # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
+        # restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
+        env:
+          HIP_LAUNCH_BLOCKING: "1"
+        run: |
+          rocminfo
+          GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-vulkan-nvidia-cm:
+    runs-on: [self-hosted, Linux, NVIDIA]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          vulkaninfo --summary
+          GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-vulkan-nvidia-cm2:
+    runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          vulkaninfo --summary
+          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-webgpu-nvidia:
+    runs-on: [self-hosted, Linux, NVIDIA, X64]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Dawn Dependency
+        id: dawn-depends
+        run: |
+          DAWN_VERSION="v20260317.182325"
+          DAWN_OWNER="google"
+          DAWN_REPO="dawn"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
+          echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+          curl -L -o artifact.tar.gz \
+            "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+          mkdir dawn
+          tar -xvf artifact.tar.gz -C dawn --strip-components=1
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          GG_BUILD_WEBGPU=1 \
+          GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
+          GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
+            bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  # TODO: provision AMX-compatible machine
+  #cpu-amx:
+  #  runs-on: [self-hosted, Linux, CPU, AMX]
+
+  #  steps:
+  #    - name: Clone
+  #      id: checkout
+  #      uses: actions/checkout@v6
+
+  #    - name: Test
+  #      id: ggml-ci
+  #      run: |
+  #        bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  # TODO: provision AMD GPU machine
+  # amd-vulkan:
+  #   runs-on: [self-hosted, Linux, AMD]
+
+  #   steps:
+  #     - name: Clone
+  #       id: checkout
+  #       uses: actions/checkout@v6
+
+  #     - name: Test
+  #       id: ggml-ci
+  #       run: |
+  #         vulkaninfo --summary
+  #         GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  # TODO: provision AMD GPU machine
+  # amd-rocm:
+  #   runs-on: [self-hosted, Linux, AMD]
+
+  #   steps:
+  #     - name: Clone
+  #       id: checkout
+  #       uses: actions/checkout@v6
+
+  #     - name: Test
+  #       id: ggml-ci
+  #       run: |
+  #         amd-smi static
+  #         GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-metal:
+    runs-on: [self-hosted, macOS, ARM64]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-webgpu-apple:
+    runs-on: [self-hosted, macOS, ARM64]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Dawn Dependency
+        id: dawn-depends
+        run: |
+          DAWN_VERSION="v20260317.182325"
+          DAWN_OWNER="google"
+          DAWN_REPO="dawn"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
+          echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+          curl -L -o artifact.tar.gz \
+            "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
+          mkdir dawn
+          tar -xvf artifact.tar.gz -C dawn --strip-components=1
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
+            bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-vulkan-apple:
+    runs-on: [self-hosted, macOS, ARM64]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          vulkaninfo --summary
+          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-vulkan-intel-linux:
+    runs-on: [self-hosted, Linux, Intel]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          vulkaninfo --summary
+          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  gpu-vulkan-intel-windows:
+    runs-on: [self-hosted, Windows, X64, Intel]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
+        env:
+          MSYSTEM: UCRT64
+          CHERE_INVOKING: 1
+          PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
+        run: |
+          vulkaninfo --summary
+          # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
+          # a valid python environment for testing
+          LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
+
+  gpu-openvino-low-perf:
+    runs-on: [self-hosted, Linux, Intel, OpenVINO]
+
+    env:
+      # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Setup OpenVINO Toolkit
+        uses: ./.github/actions/linux-setup-openvino
+        with:
+          path: ./openvino_toolkit
+          version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
+          version_full: ${{ env.OPENVINO_VERSION_FULL }}
+
+      - name: Install OpenVINO dependencies
+        run: |
+          cd ./openvino_toolkit
+          chmod +x ./install_dependencies/install_openvino_dependencies.sh
+          echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          source ./openvino_toolkit/setupvars.sh
+          GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  cpu-x64-high-perf:
+    runs-on: [self-hosted, Linux, X64]
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  cpu-arm64-high-perf-graviton4:
+    runs-on: ah-ubuntu_22_04-c8g_8x
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Dependencies
+        id: depends
+        run: |
+          set -euxo pipefail
+          sudo apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
+          apt-get install -y \
+          build-essential \
+          python3-venv \
+          gpg \
+          wget \
+          time \
+          git-lfs
+
+          git lfs install
+
+          # install the latest cmake
+          sudo install -d /usr/share/keyrings
+          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
+            | gpg --dearmor \
+            | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
+            | sudo tee /etc/apt/sources.list.d/kitware.list
+          sudo apt-get update
+          sudo apt-get install -y cmake
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+
+  cpu-arm64-graviton4-kleidiai:
+    runs-on: ah-ubuntu_22_04-c8g_8x
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Dependencies
+        id: depends
+        run: |
+          set -euxo pipefail
+          sudo apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
+          apt-get install -y \
+          build-essential \
+          python3-venv \
+          gpg \
+          wget \
+          time \
+          git-lfs
+
+          git lfs install
+
+          # install the latest cmake
+          sudo install -d /usr/share/keyrings
+          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
+            | gpg --dearmor \
+            | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
+            | sudo tee /etc/apt/sources.list.d/kitware.list
+          sudo apt-get update
+          sudo apt-get install -y cmake
+
+      - name: Test
+        id: ggml-ci
+        run: |
+          GG_BUILD_KLEIDIAI=1 \
+          GG_BUILD_EXTRA_TESTS_0=1 \
+          bash ./ci/run.sh ./tmp/results ./tmp/mnt

.github/workflows/build-sycl.yml

@@ -48,8 +48,8 @@ jobs:
 
     env:
       ONEAPI_ROOT: /opt/intel/oneapi/
-      ONEAPI_INSTALLER_VERSION: "2026.1"
-      LEVEL_ZERO_VERSION: "1.33.1"
+      ONEAPI_INSTALLER_VERSION: "2025.3.3"
+      LEVEL_ZERO_VERSION: "1.28.2"
       LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
 
     continue-on-error: true
@@ -63,32 +63,23 @@ jobs:
         shell: bash
         run: |
           cd /tmp
-          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
-          sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
+          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
+          sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
 
       - name: Install Level Zero SDK
         shell: bash
         run: |
           cd /tmp
-          # v1.33.x renamed the Debian packages to libze1 / libze-dev
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb
-          sudo apt-get install -y ./libze1.deb ./libze-dev.deb
+          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
+          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
+          sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: sycl-ubuntu-24-${{ matrix.build }}
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: sycl-ubuntu-24-${{ matrix.build }}
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -102,24 +93,18 @@ jobs:
             -DCMAKE_CXX_COMPILER=icpx \
             -DLLAMA_OPENSSL=OFF \
             -DGGML_NATIVE=OFF \
-            -DGGML_SYCL_F16=${{ matrix.fp16 }} \
-            -DGGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON \
-            -DGGML_SYCL_DNN=ON \
-            -DCMAKE_CXX_FLAGS="-fsycl-unnamed-lambda" \
-            -DCMAKE_EXE_LINKER_FLAGS="-fsycl-unnamed-lambda"
+            -DGGML_SYCL_F16=${{ matrix.fp16 }}
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
         env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+          GH_TOKEN: ${{ github.token }}
         with:
           key: sycl-ubuntu-24-${{ matrix.build }}
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
 
   windows-latest-sycl:
     runs-on: windows-2022
@@ -129,11 +114,11 @@ jobs:
         shell: bash
 
     env:
-      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
+      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
       WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
-      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
+      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
       ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
-      ONEAPI_INSTALLER_VERSION: "2026.1"
+      ONEAPI_INSTALLER_VERSION: "2025.3.3"
     steps:
       - name: Clone
         id: checkout
@@ -152,7 +137,7 @@ jobs:
           "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: sycl-windows-latest
           variant: ccache

.github/workflows/build-vulkan.yml

@@ -31,7 +31,6 @@ concurrency:
 env:
   GGML_NLOOP: 3
   GGML_N_THREADS: 1
-  GGML_SCHED_DEBUG_REALLOC: 1
   LLAMA_ARG_LOG_COLORS: 1
   LLAMA_ARG_LOG_PREFIX: 1
   LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -54,20 +53,12 @@ jobs:
           echo "CXX=g++-14" >> "$GITHUB_ENV"
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: vulkan-ubuntu-24.04-arm
           variant: ccache
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: vulkan-ubuntu-24.04-arm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Configure
         id: cmake_configure
@@ -82,17 +73,15 @@ jobs:
         run: |
           time cmake --build build -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
         env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+          GH_TOKEN: ${{ github.token }}
         with:
           key: vulkan-ubuntu-24.04-arm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
 
   ubuntu-llvmpipe:
     runs-on: ubuntu-24.04
@@ -123,19 +112,11 @@ jobs:
           strip: 1
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: vulkan-ubuntu-24.04-llvmpipe
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: vulkan-ubuntu-24.04-llvmpipe
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -146,18 +127,6 @@ jobs:
             -DGGML_VULKAN=ON
           cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: vulkan-ubuntu-24.04-llvmpipe
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
@@ -165,7 +134,19 @@ jobs:
           export GGML_VK_VISIBLE_DEVICES=0
           export GGML_VK_DISABLE_F16=1
           export GGML_VK_DISABLE_COOPMAT=1
-          ctest -L main --verbose --timeout 900
+          # This is using llvmpipe and runs slower than other backends
+          # test-backend-ops is too slow on llvmpipe, skip it
+          ctest -L main -E test-backend-ops --verbose --timeout 900
+
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: vulkan-ubuntu-24.04-llvmpipe
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
 
   windows:
     runs-on: windows-2025
@@ -179,7 +160,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cpu-windows-2025-x64-vulkan
           variant: ccache

.github/workflows/build-wasm.yml

@@ -54,10 +54,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: webgpu-ubuntu-24.04-arm-wasm
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Install Emscripten
         run: |
@@ -68,22 +69,13 @@ jobs:
 
       - name: Fetch emdawnwebgpu
         run: |
-          DAWN_TAG="v20260908.214631"
+          DAWN_TAG="v20260317.182325"
           EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip"
           echo "Downloading ${EMDAWN_PKG}"
           curl -L -o emdawn.zip \
             "https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"
           unzip emdawn.zip
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: webgpu-ubuntu-24.04-arm-wasm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build WASM WebGPU
         run: |
           source emsdk/emsdk_env.sh
@@ -97,14 +89,12 @@ jobs:
 
           time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
         env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+          GH_TOKEN: ${{ github.token }}
         with:
           key: webgpu-ubuntu-24.04-arm-wasm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/build-webgpu.yml

@@ -69,33 +69,25 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: webgpu-macos-latest
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Dawn Dependency
         id: dawn-depends
         run: |
-          DAWN_VERSION="v20260908.214631"
+          DAWN_VERSION="v20260317.182325"
           DAWN_OWNER="google"
           DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
           echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           curl -L -o artifact.tar.gz \
             "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           mkdir dawn
           tar -xvf artifact.tar.gz -C dawn --strip-components=1
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: webgpu-macos-latest
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
@@ -103,24 +95,22 @@ jobs:
           cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
           time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: webgpu-macos-latest
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
           cd build
           ctest -L main --verbose --timeout 900
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: webgpu-macos-latest
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+
   ubuntu:
     runs-on: ubuntu-24.04
 
@@ -130,10 +120,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: webgpu-ubuntu-24.04
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Dependencies
         id: depends
@@ -147,25 +138,16 @@ jobs:
         id: dawn-depends
         run: |
           sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
-          DAWN_VERSION="v20260908.214631"
+          DAWN_VERSION="v20260317.182325"
           DAWN_OWNER="google"
           DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
           echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           curl -L -o artifact.tar.gz \
             "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           mkdir dawn
           tar -xvf artifact.tar.gz -C dawn --strip-components=1
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: webgpu-ubuntu-24.04
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
@@ -174,20 +156,20 @@ jobs:
             -DGGML_WEBGPU=ON
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: webgpu-ubuntu-24.04
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
           cd build
-          ctest -L main --verbose --timeout 900
+          # This is using llvmpipe and runs slower than other backends
+          # test-backend-ops is too slow on llvmpipe, skip it
+          ctest -L main -E test-backend-ops --verbose --timeout 900
+
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: webgpu-ubuntu-24.04
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/check-vendor.yml

@@ -19,7 +19,7 @@ on:
 
 jobs:
   check-vendor:
-    runs-on: ubuntu-slim
+    runs-on: [self-hosted, fast]
 
     steps:
       - name: Checkout

.github/workflows/ci-self-hosted-cpu.yml

@@ -1,112 +0,0 @@
-name: CI (self-hosted CPU backend)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/ci-self-hosted-cpu.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/ci-self-hosted-cpu.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      'ggml/src/*',
-      'ggml/src/ggml-cpu/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  cpu-x64-high-perf:
-    runs-on: [self-hosted, Linux, X64]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-  cpu-arm64-high-perf-graviton4:
-    runs-on: ah-ubuntu_24_04-c8g_8x
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Dependencies
-        id: depends
-        run: |
-          set -euxo pipefail
-          sudo apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
-          apt-get install -y \
-          build-essential \
-          python3-venv \
-          gpg \
-          wget \
-          time \
-          git-lfs
-
-          git lfs install
-
-          # install the latest cmake
-          sudo install -d /usr/share/keyrings
-          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
-            | gpg --dearmor \
-            | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
-          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
-            | sudo tee /etc/apt/sources.list.d/kitware.list
-          sudo apt-get update
-          sudo apt-get install -y cmake
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          LLAMA_ARG_THREADS=$(nproc) \
-          GG_BUILD_HIGH_PERF=1 \
-          GG_BUILD_NO_BF16=1 \
-          GG_BUILD_EXTRA_TESTS_0=1 \
-          bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-  # TODO: provision AMX-compatible machine
-  #cpu-amx:
-  #  runs-on: [self-hosted, Linux, CPU, AMX]
-
-  #  steps:
-  #    - name: Clone
-  #      id: checkout
-  #      uses: actions/checkout@v6
-
-  #    - name: Test
-  #      id: ggml-ci
-  #      run: |
-  #        bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/ci-self-hosted-cuda.yml

@@ -1,124 +0,0 @@
-name: CI (self-hosted CUDA backend)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/ci-self-hosted-cuda.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp',
-      '**/*.cu',
-      '**/*.cuh'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/ci-self-hosted-cuda.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      'ggml/src/*',
-      'ggml/src/ggml-cpu/**',
-      'ggml/src/ggml-cuda/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  gpu-cuda:
-    runs-on: "hf-jobs-t4-small:cuda13"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: self-hosted-gpu-cuda
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          nvidia-smi
-          GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: self-hosted-gpu-cuda
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-  gpu-rocm:
-    runs-on: [self-hosted, Linux, AMD]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Test
-        id: ggml-ci
-        # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
-        # issue on integrated RDNA3.5 (gfx1151) where batched inference returns
-        # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
-        # restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
-        env:
-          HIP_LAUNCH_BLOCKING: "1"
-        run: |
-          rocminfo
-          GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-  # TODO: provision AMD GPU machine
-  # amd-rocm:
-  #   runs-on: [self-hosted, Linux, AMD]
-
-  #   steps:
-  #     - name: Clone
-  #       id: checkout
-  #       uses: actions/checkout@v6
-
-  #     - name: Test
-  #       id: ggml-ci
-  #       run: |
-  #         amd-smi static
-  #         GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/ci-self-hosted-kleidiai.yml

@@ -1,85 +0,0 @@
-name: CI (self-hosted KleidiAI backend)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/ci-self-hosted-kleidiai.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/ci-self-hosted-kleidiai.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      'ggml/src/*',
-      'ggml/src/ggml-cpu/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  cpu-arm64-graviton4-kleidiai:
-    runs-on: ah-ubuntu_24_04-c8g_8x
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Dependencies
-        id: depends
-        run: |
-          set -euxo pipefail
-          sudo apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
-          apt-get install -y \
-          build-essential \
-          python3-venv \
-          gpg \
-          wget \
-          time \
-          git-lfs
-
-          git lfs install
-
-          # install the latest cmake
-          sudo install -d /usr/share/keyrings
-          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
-            | gpg --dearmor \
-            | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
-          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
-            | sudo tee /etc/apt/sources.list.d/kitware.list
-          sudo apt-get update
-          sudo apt-get install -y cmake
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          LLAMA_ARG_THREADS=$(nproc) \
-          GG_BUILD_KLEIDIAI=1 \
-          GG_BUILD_EXTRA_TESTS_0=1 \
-          GG_BUILD_HIGH_PERF=1 \
-          bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/ci-self-hosted-metal.yml

@@ -1,59 +0,0 @@
-name: CI (self-hosted Metal backend)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/ci-self-hosted-metal.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp',
-      '**/*.swift',
-      '**/*.m',
-      '**/*.metal'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/ci-self-hosted-metal.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      'ggml/src/*',
-      'ggml/src/ggml-cpu/**',
-      'ggml/src/ggml-metal/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  gpu-metal:
-    runs-on: [self-hosted, macOS, ARM64]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/ci-self-hosted-openvino.yml

@@ -1,75 +0,0 @@
-name: CI (self-hosted OpenVINO backend)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/ci-self-hosted-openvino.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/ci-self-hosted-openvino.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      'ggml/src/*',
-      'ggml/src/ggml-cpu/**',
-      'ggml/src/ggml-openvino/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  gpu-openvino-low-perf:
-    runs-on: [self-hosted, Linux, Intel, OpenVINO]
-
-    env:
-      # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.4"
-      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Setup OpenVINO Toolkit
-        uses: ./.github/actions/linux-setup-openvino
-        with:
-          path: ./openvino_toolkit
-          version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
-          version_full: ${{ env.OPENVINO_VERSION_FULL }}
-
-      - name: Install OpenVINO dependencies
-        run: |
-          cd ./openvino_toolkit
-          chmod +x ./install_dependencies/install_openvino_dependencies.sh
-          echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          source ./openvino_toolkit/setupvars.sh
-          GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/ci-self-hosted-vulkan.yml

@@ -1,201 +0,0 @@
-name: CI (self-hosted Vulkan backend)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/ci-self-hosted-vulkan.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp',
-      '**/*.comp',
-      '**/*.glsl'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/ci-self-hosted-vulkan.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      'ggml/src/*',
-      'ggml/src/ggml-cpu/**',
-      'ggml/src/ggml-vulkan/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  gpu-vulkan-nvidia-cm:
-    # runs-on: "hf-jobs-t4-small:ubuntu26_04"
-    runs-on: [self-hosted, Linux, NVIDIA]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      # - name: Install dependencies
-      #   run: |
-      #     sudo apt update
-      #     sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
-
-      # - name: ccache
-      #   uses: ggml-org/ccache-action@v1.2.24
-      #   with:
-      #     restore: false
-      #     save: false
-
-      # - name: ccache-buckets-restore
-      #   uses: ./.github/actions/ccache-buckets
-      #   with:
-      #     key: self-hosted-vulkan-nvidia-cm
-      #     folder: llama.cpp
-      #     hf_bucket: ggml-org/cache
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          vulkaninfo --summary
-          GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-      # - name: ccache-buckets-save
-      #   if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-      #   uses: ./.github/actions/ccache-buckets
-      #   env:
-      #     HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-      #   with:
-      #     key: self-hosted-vulkan-nvidia-cm
-      #     folder: llama.cpp
-      #     evict-old-files: 1d
-      #     hf_bucket: ggml-org/cache
-      #     save: true
-
-  gpu-vulkan-nvidia-cm2:
-    # runs-on: "hf-jobs-t4-small:ubuntu26_04"
-    runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      # - name: Install dependencies
-      #   run: |
-      #     sudo apt update
-      #     sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
-
-      # - name: ccache
-      #   uses: ggml-org/ccache-action@v1.2.24
-      #   with:
-      #     restore: false
-      #     save: false
-
-      # - name: ccache-buckets-restore
-      #   uses: ./.github/actions/ccache-buckets
-      #   with:
-      #     key: self-hosted-vulkan-nvidia-cm2
-      #     folder: llama.cpp
-      #     hf_bucket: ggml-org/cache
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          vulkaninfo --summary
-          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-      # - name: ccache-buckets-save
-      #   if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-      #   uses: ./.github/actions/ccache-buckets
-      #   env:
-      #     HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-      #   with:
-      #     key: self-hosted-vulkan-nvidia-cm2
-      #     folder: llama.cpp
-      #     evict-old-files: 1d
-      #     hf_bucket: ggml-org/cache
-      #     save: true
-
-  gpu-vulkan-apple:
-    runs-on: [self-hosted, macOS, ARM64]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          vulkaninfo --summary
-          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-  gpu-vulkan-intel-linux:
-    runs-on: [self-hosted, Linux, Intel]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          vulkaninfo --summary
-          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-  gpu-vulkan-intel-windows:
-    runs-on: [self-hosted, Windows, X64, Intel]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Test
-        id: ggml-ci
-        shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
-        env:
-          MSYSTEM: UCRT64
-          CHERE_INVOKING: 1
-          PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
-        run: |
-          vulkaninfo --summary
-          # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
-          # a valid python environment for testing
-          LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
-
-  # TODO: provision AMD GPU machine
-  # amd-vulkan:
-  #   runs-on: [self-hosted, Linux, AMD]
-
-  #   steps:
-  #     - name: Clone
-  #       id: checkout
-  #       uses: actions/checkout@v6
-
-  #     - name: Test
-  #       id: ggml-ci
-  #       run: |
-  #         vulkaninfo --summary
-  #         GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/ci-self-hosted-webgpu.yml

@@ -1,130 +0,0 @@
-name: CI (self-hosted WebGPU backend)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/ci-self-hosted-webgpu.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp',
-      '**/*.wgsl'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/ci-self-hosted-webgpu.yml',
-      'ci/run.sh',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      'ggml/src/*',
-      'ggml/src/ggml-cpu/**',
-      'ggml/src/ggml-webgpu/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  gpu-webgpu-nvidia:
-    runs-on: "hf-jobs-t4-small:ubuntu26_04"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: self-hosted-webgpu-nvidia
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Dawn Dependency
-        id: dawn-depends
-        run: |
-          DAWN_VERSION="v20260908.214631"
-          DAWN_OWNER="google"
-          DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
-          echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          curl -L -o artifact.tar.gz \
-            "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          mkdir dawn
-          tar -xvf artifact.tar.gz -C dawn --strip-components=1
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          GG_BUILD_WEBGPU=1 \
-          GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
-          GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
-            bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: self-hosted-webgpu-nvidia
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-  gpu-webgpu-apple:
-    runs-on: [self-hosted, macOS, ARM64]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Dawn Dependency
-        id: dawn-depends
-        run: |
-          DAWN_VERSION="v20260908.214631"
-          DAWN_OWNER="google"
-          DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
-          echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          curl -L -o artifact.tar.gz \
-            "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          mkdir dawn
-          tar -xvf artifact.tar.gz -C dawn --strip-components=1
-
-      - name: Test
-        id: ggml-ci
-        run: |
-          GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
-            bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/code-style.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   model-naming:
-    runs-on: ubuntu-slim
+    runs-on: [self-hosted, fast]
     steps:
       - uses: actions/checkout@v6
       - name: Check model naming conventions

.github/workflows/copilot-setup-steps.yml

@@ -11,12 +11,10 @@ on:
     paths:
       - .github/workflows/copilot-setup-steps.yml
 
-cache-mode: none
-
 jobs:
   # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
   copilot-setup-steps:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
 
     # Set the permissions to the lowest permissions possible needed for your steps.
     # Copilot will be given its own token for its operations.
@@ -31,10 +29,10 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
-          restore: false
-          save: false
+          key: copilot-setup-steps
+          evict-old-files: 1d
 
       - name: Dependencies
         id: depends

.github/workflows/docker.yml

@@ -90,8 +90,8 @@ jobs:
             { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true },
             { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
             { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
-            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
-            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
+            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
+            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.3.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
             { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
             { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
             { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },

.github/workflows/editorconfig.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   editorconfig:
-    runs-on: ubuntu-slim
+    runs-on: [self-hosted, fast]
     steps:
       - uses: actions/checkout@v6
       - uses: editorconfig-checker/action-editorconfig-checker@840e866d93b8e032123c23bac69dece044d4d84c # v2.2.0

.github/workflows/gguf-publish.yml

@@ -21,7 +21,7 @@ on:
 jobs:
   deploy:
 
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/hip-quality-check.yml

@@ -49,22 +49,14 @@ jobs:
         id: depends
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3 python3-venv python3-pip jq
+          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: hip-quality-check-ubuntu-22.04
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: hip-quality-check-ubuntu-22.04
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build with Werror
         id: cmake_build
@@ -93,14 +85,12 @@ jobs:
           make -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$'
           python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
         env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+          GH_TOKEN: ${{ github.token }}
         with:
           key: hip-quality-check-ubuntu-22.04
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/make-release.yml

@@ -13,27 +13,16 @@ on:
         required: true
         type: boolean
         default: true
-      skip_apiabi_check:
-        description: 'Skip API/ABI compatibility check'
-        required: false
-        type: boolean
-        default: false
-      apiabi_compare_tag:
-        description: 'Tag to compare against for API/ABI check (default: latest release)'
-        required: false
-        type: string
-        default: ''
 
 env:
   GH_TOKEN: ${{ github.token }}
 
 permissions:
   contents: write
-  packages: write
 
 jobs:
   make-release:
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
@@ -43,18 +32,12 @@ jobs:
           ref: ${{ inputs.commit != '' && inputs.commit || github.ref_name }}
           fetch-depth: 0
 
-      - name: Install API/ABI check tools
-        if: ${{ github.event.inputs.skip_apiabi_check != 'true' }}
-        run: sudo apt-get install -y abi-compliance-checker abigail-tools
-
       - name: Run release checks
         id: checks
         run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }}
         env:
           GITHUB_REPOSITORY: ${{ github.repository }}
           RELEASE_BRANCH: ${{ github.ref_name }}
-          SKIP_APIABI_CHECK: ${{ github.event.inputs.skip_apiabi_check }}
-          APIABI_COMPARE_TAG: ${{ github.event.inputs.apiabi_compare_tag }}
 
       - name: Create release tag
         if: ${{ github.event.inputs.dry_run == 'false' }}
@@ -130,29 +113,6 @@ jobs:
               data: await fs.readFileSync('./nightly-tag.txt')
             });
 
-      - name: Re-tag container images with release version
-        if: ${{ github.event.inputs.dry_run == 'false' && steps.desc.outputs.nightly_tag != '' }}
-        env:
-          GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
-        run: |
-          VERSION="${{ steps.checks.outputs.version }}"
-          NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}"
-          REPO_OWNER="${GITHUB_REPOSITORY_OWNER,,}"
-          IMAGE_REPO="ghcr.io/${REPO_OWNER}/${{ github.event.repository.name }}"
-
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
-
-          VARIANTS=("" "-cuda" "-cuda13" "-vulkan" "-rocm" "-intel" "-musa" "-openvino")
-          TYPES=("full" "light" "server")
-          for type in "${TYPES[@]}"; do
-            for variant in "${VARIANTS[@]}"; do
-              src="${IMAGE_REPO}:${type}${variant}-${NIGHTLY_TAG}"
-              dst="${IMAGE_REPO}:${type}${variant}-${VERSION}"
-              echo "Tagging ${src} -> ${dst}"
-              docker buildx imagetools create --tag "${dst}" "${src}"
-            done
-          done
-
       - name: Dry run summary
         if: ${{ github.event.inputs.dry_run == 'true' }}
         run: |

.github/workflows/models-check.yml

@@ -1,445 +0,0 @@
-name: Models Backend Check
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/models-check.yml',
-      'ggml/**',
-      'tests/fusion/**',
-      'tests/test-fusion.cpp',
-      'tests/test-llama-archs.cpp',
-      'src/llama-graph.cpp',
-      'src/llama-model*',
-      'src/models/**'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/models-check.yml',
-      'ggml/**',
-      'tests/fusion/**',
-      'tests/test-fusion.cpp',
-      'tests/test-llama-archs.cpp',
-      'src/llama-graph.cpp',
-      'src/llama-model*',
-      'src/models/**'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  cuda:
-    runs-on: "hf-jobs-t4-medium:cuda13"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y cmake time python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: models-check-cuda
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DGGML_SCHED_NO_REALLOC=ON \
-            -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \
-            -DGGML_CUDA=ON
-          time cmake --build build --config Release --target test-llama-archs -j$(nproc)
-          time cmake --build build --config Release --target test-fusion -j$(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: models-check-cuda
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      # - name: Generate models
-      #   id: generate_models
-      #   run: |
-      #     rm -rf build-ci-models && mkdir -p build-ci-models
-      #     ./build/bin/test-llama-archs -o build-ci-models
-
-      # TODO: add for backends as they adopt the fusion debug API
-      # - name: Test fusion
-      #   id: test_fusion
-      #   run: |
-      #     ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv
-
-      - name: Test archs
-        id: test_archs
-        run: |
-          GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1
-          GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1
-          GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1
-          GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1
-
-  metal:
-    runs-on: [self-hosted, macOS, ARM64]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DGGML_SCHED_NO_REALLOC=ON \
-            -DGGML_BLAS=OFF \
-            -DGGML_METAL=ON
-          time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)
-          time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)
-
-      - name: Generate models
-        id: generate_models
-        run: |
-          rm -rf build-ci-models && mkdir -p build-ci-models
-          ./build/bin/test-llama-archs -o build-ci-models
-
-      - name: Test fusion
-        id: test_fusion
-        run: |
-          ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv
-
-      - name: Test archs
-        id: test_archs
-        run: |
-          GGML_METAL_DEVICES=1 ./build/bin/test-llama-archs -s 1
-          GGML_METAL_DEVICES=2 ./build/bin/test-llama-archs -s 1
-          GGML_METAL_DEVICES=3 ./build/bin/test-llama-archs -s 1
-          GGML_METAL_DEVICES=4 ./build/bin/test-llama-archs -s 1
-
-  rocm:
-    runs-on: [self-hosted, Linux, gfx1201]
-    container: "rocm/dev-ubuntu-24.04:7.2.4-complete"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Install dependencies
-        run: |
-          apt update
-          apt install -y build-essential jq cmake time python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: models-check-rocm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DGGML_SCHED_NO_REALLOC=ON \
-            -DCMAKE_HIP_COMPILER=$(hipconfig -l)/clang \
-            -DGPU_TARGETS=gfx1201 \
-            -DGGML_HIP=ON
-          time cmake --build build --config Release --target test-llama-archs -j$(nproc)
-          time cmake --build build --config Release --target test-fusion -j$(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: models-check-rocm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      # - name: Generate models
-      #   id: generate_models
-      #   run: |
-      #     rm -rf build-ci-models && mkdir -p build-ci-models
-      #     ./build/bin/test-llama-archs -o build-ci-models
-
-      # TODO: add for backends as they adopt the fusion debug API
-      # - name: Test fusion
-      #   id: test_fusion
-      #   run: |
-      #     ./build/bin/test-fusion --models build-ci-models --device CUDA0 --check tests/fusion/CUDA.csv
-
-      - name: Test archs
-        id: test_archs
-        run: |
-          GGML_CUDA_DEVICES=1 ./build/bin/test-llama-archs -s 1
-          GGML_CUDA_DEVICES=2 ./build/bin/test-llama-archs -s 1
-          GGML_CUDA_DEVICES=3 ./build/bin/test-llama-archs -s 1
-          GGML_CUDA_DEVICES=4 ./build/bin/test-llama-archs -s 1
-
-  vulkan-nvidia:
-    runs-on: "hf-jobs-t4-small:ubuntu26_04"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: models-check-vulkan-nvidia
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DGGML_SCHED_NO_REALLOC=ON \
-            -DGGML_VULKAN=ON
-          time cmake --build build --config Release --target test-llama-archs -j$(nproc)
-          time cmake --build build --config Release --target test-fusion -j$(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: models-check-vulkan-nvidia
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      # - name: Generate models
-      #   id: generate_models
-      #   run: |
-      #     rm -rf build-ci-models && mkdir -p build-ci-models
-      #     ./build/bin/test-llama-archs -o build-ci-models
-
-      # TODO: add for backends as they adopt the fusion debug API
-      # - name: Test fusion
-      #   id: test_fusion
-      #   run: |
-      #     ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
-
-      - name: Test archs
-        id: test_archs
-        run: |
-          ./build/bin/test-llama-archs -s 1
-
-  vulkan-amd:
-    runs-on: [self-hosted, Linux, gfx1201]
-    container: "ubuntu:26.04"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Install dependencies
-        run: |
-          apt update
-          apt install -y build-essential jq cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan-dev glslc spirv-headers vulkan-tools mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: models-check-vulkan-amd
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DGGML_SCHED_NO_REALLOC=ON \
-            -DGGML_VULKAN=ON
-          time cmake --build build --config Release --target test-llama-archs -j$(nproc)
-          time cmake --build build --config Release --target test-fusion -j$(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: models-check-vulkan-amd
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      # - name: Generate models
-      #   id: generate_models
-      #   run: |
-      #     rm -rf build-ci-models && mkdir -p build-ci-models
-      #     ./build/bin/test-llama-archs -o build-ci-models
-
-      # TODO: add for backends as they adopt the fusion debug API
-      # - name: Test fusion
-      #   id: test_fusion
-      #   run: |
-      #     ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
-
-      - name: Test archs
-        id: test_archs
-        run: |
-          ./build/bin/test-llama-archs -s 1
-
-  webgpu-nvidia:
-    runs-on: "hf-jobs-t4-small:ubuntu26_04"
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 time python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: models-check-webgpu-nvidia
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Dawn Dependency
-        id: dawn-depends
-        run: |
-          DAWN_VERSION="v20260908.214631"
-          DAWN_OWNER="google"
-          DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
-          echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          curl -L -o artifact.tar.gz \
-            "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
-          mkdir dawn
-          tar -xvf artifact.tar.gz -C dawn --strip-components=1
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DGGML_SCHED_NO_REALLOC=ON \
-            -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/dawn" \
-            -DDawn_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
-            -DGGML_WEBGPU=ON
-          time cmake --build build --config Release --target test-llama-archs -j$(nproc)
-          time cmake --build build --config Release --target test-fusion -j$(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: models-check-webgpu-nvidia
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      # - name: Generate models
-      #   id: generate_models
-      #   run: |
-      #     rm -rf build-ci-models && mkdir -p build-ci-models
-      #     ./build/bin/test-llama-archs -o build-ci-models
-
-      # TODO: add for backends as they adopt the fusion debug API
-      # - name: Test fusion
-      #   id: test_fusion
-      #   run: |
-      #     ./build/bin/test-fusion --models build-ci-models --device WebGPU --check tests/fusion/WebGPU.csv
-
-      - name: Test archs
-        id: test_archs
-        run: |
-          ./build/bin/test-llama-archs -s 1

.github/workflows/pre-tokenizer-hashes.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
     pre-tokenizer-hashes:
-        runs-on: ubuntu-slim
+        runs-on: [self-hosted, fast]
 
         steps:
         - name: Checkout repository

.github/workflows/python-check-requirements.yml

@@ -20,7 +20,7 @@ concurrency:
 
 jobs:
   python-check-requirements:
-    runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
+    runs-on: [self-hosted, CPU, fast]
     name: check-requirements
     steps:
       - name: Check out source repository

.github/workflows/python-lint.yml

@@ -21,7 +21,7 @@ concurrency:
 
 jobs:
   flake8-lint:
-    runs-on: ubuntu-slim
+    runs-on: [self-hosted, fast]
     name: Lint
     steps:
       - name: Check out source repository

.github/workflows/python-type-check.yml

@@ -22,7 +22,7 @@ concurrency:
 
 jobs:
   python-type-check:
-    runs-on: ubuntu-slim
+    runs-on: [self-hosted, fast]
     name: python type-check
     steps:
       - name: Check out source repository
@@ -31,7 +31,7 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.11"
-          pip-install: -r requirements/requirements-all.txt ty==0.0.84
+          pip-install: -r requirements/requirements-all.txt ty==0.0.35
       # - name: Type-check with Pyright
       #   uses: jakebailey/pyright-action@v2
       #   with:
@@ -40,4 +40,4 @@ jobs:
       #     warnings: true
       - name: Type-check with ty
         run: |
-            ty check --exit-zero-on-warning --output-format=github
+            ty check --output-format=github

.github/workflows/release.yml

@@ -103,10 +103,9 @@ jobs:
           path: tools/ui/dist
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-${{ matrix.os }}-${{ matrix.arch }}
-          evict-old-files: 1d
 
       - name: Build
         id: cmake_build
@@ -188,10 +187,9 @@ jobs:
 
       - name: ccache
         if: ${{ matrix.build != 's390x' }}
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-${{ matrix.os }}-cpu
-          evict-old-files: 1d
 
       - name: Build
         id: cmake_build
@@ -274,10 +272,9 @@ jobs:
           fi
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-${{ matrix.os }}-vulkan
-          evict-old-files: 1d
 
       - name: Build
         id: cmake_build
@@ -313,151 +310,11 @@ jobs:
         with:
           key: release-${{ matrix.os }}-vulkan
 
-  ubuntu-cuda:
-    name: ubuntu-cuda (${{ matrix.label }}, ${{ matrix.build }})
-    needs: [check-release, ui-build]
-    if: ${{ needs.check-release.outputs.should_release == 'true' }}
-
-    strategy:
-      matrix:
-        include:
-          # label = short version used in artifact names / release body
-          # cuda  = full container image tag
-          # CTK >= 13.5 bundles CCCL >= 3.5; omit GGML_CUDA_CCCL_VERSION for those versions.
-          - build: 'x64'
-            os: ubuntu-24.04
-            cuda: '12.8.2'
-            label: '12.8'
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
-          - build: 'x64'
-            os: ubuntu-24.04
-            cuda: '13.4.1'
-            label: '13.4'
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
-          - build: 'arm64'
-            os: ubuntu-24.04-arm
-            cuda: '13.4.1'
-            label: '13.4'
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
-
-    runs-on: ${{ matrix.os }}
-    container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04
-
-    permissions:
-      actions: write
-
-    steps:
-      # the container has no git; install it before checkout so that a real git
-      # repository is created (the get-tag-name action and the build both need it)
-      - name: Install git
-        run: |
-          apt-get update
-          apt-get install -y --no-install-recommends git
-
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      # checkout runs as the host user; in-container steps run as root, so git
-      # refuses to touch a repo it does not own. Mark the workspace as safe.
-      # use the env var: the github.workspace context holds the HOST path,
-      # GITHUB_WORKSPACE the container path
-      - name: Git safe directory
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
-
-      - name: Download UI build
-        uses: actions/download-artifact@v7
-        with:
-          name: llama-ui.zip
-          path: tools/ui/dist
-
-      - name: Dependencies
-        id: depends
-        # container jobs default to sh (dash); need bash for the [[ ]] below
-        shell: bash
-        run: |
-          apt-get update
-          apt-get install -y --no-install-recommends build-essential cmake ninja-build libssl-dev jq python3-venv
-          # the container ships GCC 13, which does not know the 'sme' march
-          # feature used by the armv9.2 CPU variant of GGML_CPU_ALL_VARIANTS
-          if [[ "${{ matrix.build }}" == "arm64" ]]; then
-            apt-get install -y --no-install-recommends gcc-14 g++-14
-            echo "CC=gcc-14" >> "$GITHUB_ENV"
-            echo "CXX=g++-14" >> "$GITHUB_ENV"
-          fi
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }}
-          evict-old-files: 1d
-          max-size: "1G"
-
-      - name: Build
-        id: cmake_build
-        # no CMAKE_CUDA_ARCHITECTURES: use the broad default arch set from
-        # ggml/src/ggml-cuda/CMakeLists.txt so the release binary covers many GPUs
-        run: |
-          cmake -B build \
-            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DGGML_BACKEND_DL=ON \
-            -DGGML_NATIVE=OFF \
-            -DGGML_CPU_ALL_VARIANTS=ON \
-            -DGGML_CUDA=ON \
-            -DGGML_CUDA_NCCL=OFF \
-            ${{ env.CMAKE_ARGS }} ${{ matrix.defines }}
-          cmake --build build --config Release -j $(nproc)
-
-      - name: Determine tag name
-        id: tag
-        uses: ./.github/actions/get-tag-name
-
-      - name: Pack artifacts
-        id: pack_artifacts
-        run: |
-          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C ./build/bin .
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v6
-        with:
-          path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
-          name: llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
-
-      # ship the CUDA runtime libraries the backend links against, mirroring
-      # the windows-cuda cudart zip - extract next to the binaries ($ORIGIN rpath)
-      - name: Pack CUDA runtime
-        id: pack_cuda_runtime
-        run: |
-          major="${{ matrix.label }}"
-          major="${major%%.*}"
-          mkdir -p ./cudart
-          # cp -L dereferences the SONAME symlinks into plain files, so the
-          # tarball holds exactly 3 files with no versioned duplicates
-          cp -L /usr/local/cuda/lib64/libcudart.so.${major} ./cudart/
-          cp -L /usr/local/cuda/lib64/libcublas.so.${major} ./cudart/
-          cp -L /usr/local/cuda/lib64/libcublasLt.so.${major} ./cudart/
-          tar -czvf cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz --transform "s,^\.,cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}," -C ./cudart .
-
-      - name: Upload CUDA runtime
-        uses: actions/upload-artifact@v6
-        with:
-          path: cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
-          name: cudart-llama-bin-ubuntu-cuda-${{ matrix.label }}-${{ matrix.build }}.tar.gz
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        with:
-          key: release-ubuntu-${{ matrix.os }}-cuda-${{ matrix.label }}-${{ matrix.build }}
-
   android-arm64:
     needs: [check-release, ui-build]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
 
     #permissions:
     #  actions: write
@@ -485,7 +342,7 @@ jobs:
           distribution: temurin
 
       - name: Setup Android SDK
-        uses: android-actions/setup-android@be39fa834029ff78f1a44aa3bb0819b8fc2bd8fd # v4.0.4
+        uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
         with:
           log-accepted-android-sdk-licenses: false
 
@@ -501,10 +358,9 @@ jobs:
       #   cache on:  https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
       #
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.21
       #  with:
       #    key: release-android-arm64
-      #    evict-old-files: 1d
 
       - name: Build
         id: cmake_build
@@ -545,120 +401,6 @@ jobs:
           path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz
           name: llama-bin-android-arm64.tar.gz
 
-  android-arm64-snapdragon:
-    needs: [check-release, ui-build]
-    if: ${{ needs.check-release.outputs.should_release == 'true' }}
-
-    runs-on: ubuntu-latest
-    container: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
-
-    defaults:
-      run:
-        shell: bash
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      # checkout runs as the host user; in-container steps run as root, so git
-      # refuses to touch a repo it does not own. Mark the workspace as safe.
-      - name: Git safe directory
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
-
-      - name: Download UI build
-        uses: actions/download-artifact@v7
-        with:
-          name: llama-ui.zip
-          path: tools/ui/dist
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cp docs/backend/snapdragon/CMakeUserPresets.json .
-          cmake --preset arm64-android-snapdragon-release -B build \
-            -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DLLAMA_BUILD_BORINGSSL=ON \
-            ${{ env.CMAKE_ARGS }}
-          cmake --build build -j $(nproc)
-          cmake --install build --prefix pkg-snapdragon/llama.cpp
-
-      - name: Determine tag name
-        id: tag
-        uses: ./.github/actions/get-tag-name
-
-      - name: Pack artifacts
-        id: pack_artifacts
-        run: |
-          cp LICENSE pkg-snapdragon/llama.cpp/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C pkg-snapdragon/llama.cpp .
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v6
-        with:
-          path: llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz
-          name: llama-bin-android-arm64-snapdragon.tar.gz
-
-  linux-arm64-snapdragon:
-    needs: [check-release, ui-build]
-    if: ${{ needs.check-release.outputs.should_release == 'true' }}
-
-    runs-on: ubuntu-latest
-    container: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
-
-    defaults:
-      run:
-        shell: bash
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      # checkout runs as the host user; in-container steps run as root, so git
-      # refuses to touch a repo it does not own. Mark the workspace as safe.
-      - name: Git safe directory
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
-
-      - name: Download UI build
-        uses: actions/download-artifact@v7
-        with:
-          name: llama-ui.zip
-          path: tools/ui/dist
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cp docs/backend/snapdragon/CMakeUserPresets.json .
-          cmake --preset arm64-linux-snapdragon-release -B build -DGGML_OPENCL=ON \
-            -DCMAKE_INSTALL_RPATH='$ORIGIN;$ORIGIN/../lib' \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DLLAMA_BUILD_BORINGSSL=ON \
-            ${{ env.CMAKE_ARGS }}
-          cmake --build build -j $(nproc)
-          cmake --install build --prefix pkg-snapdragon/llama.cpp
-
-      - name: Determine tag name
-        id: tag
-        uses: ./.github/actions/get-tag-name
-
-      - name: Pack artifacts
-        id: pack_artifacts
-        run: |
-          cp LICENSE pkg-snapdragon/llama.cpp/
-          tar -czvf llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz --transform "s,^\.,llama-${{ steps.tag.outputs.name }}," -C pkg-snapdragon/llama.cpp .
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v6
-        with:
-          path: llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz
-          name: llama-bin-linux-arm64-snapdragon.tar.gz
-
   ubuntu-24-openvino:
     needs: [check-release, ui-build]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -673,8 +415,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.4"
-      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Set OpenVINO version output
@@ -694,10 +436,9 @@ jobs:
           path: tools/ui/dist
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-ubuntu-24.04-openvino-release-no-preset-v1
-          evict-old-files: 1d
 
       - name: Dependencies
         run: |
@@ -788,8 +529,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.4"
-      OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Set OpenVINO version output
@@ -810,7 +551,7 @@ jobs:
           path: tools/ui/dist
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-openvino
           variant: ccache
@@ -938,10 +679,9 @@ jobs:
           choco install ninja
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
-          evict-old-files: 1d
 
       - name: Build
         shell: cmd
@@ -985,7 +725,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - ROCM_VERSION: "10.0.0"
+          - ROCM_VERSION: "7.14.0"
             gpu_targets: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201"
             build: x64
 
@@ -1001,7 +741,7 @@ jobs:
           choco install ninja
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
           evict-old-files: 1d
@@ -1183,10 +923,9 @@ jobs:
 
       # TODO: these jobs need to use llvm toolchain in order to utilize the ccache
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.21
       #  with:
       #    key: release-windows-2025-${{ matrix.arch }}-${{ matrix.backend }}
-      #    evict-old-files: 1d
 
       - name: Install OpenCL Headers and Libs
         id: install_opencl
@@ -1245,16 +984,15 @@ jobs:
     strategy:
       matrix:
         include:
-          # CTK >= 13.5 bundles CCCL >= 3.5; omit GGML_CUDA_CCCL_VERSION for those versions.
           - cuda: '12.4'
             arch: x64
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
-          - cuda: '13.4'
+            defines: '-DGGML_CUDA_CUB_3DOT2=ON'
+          - cuda: '13.3'
             arch: x64
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
+            defines: ''
           - cuda: '13.4'
             arch: arm64
-            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
+            defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
 
     steps:
       - name: Clone
@@ -1273,14 +1011,14 @@ jobs:
           choco install ninja
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
-          evict-old-files: 1d
 
       - name: Build
         id: cmake_build
         shell: cmd
+        # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
         run: |
           call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
           cmake -S . -B build -G "Ninja Multi-Config" ^
@@ -1345,11 +1083,11 @@ jobs:
         shell: bash
 
     env:
-      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0cb67a0d-67f6-410b-868b-f4a0a17ff0cf/intel-oneapi-toolkit-2026.1.1.32_offline.exe
+      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
       WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
-      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.33.1/level-zero-win-sdk-1.33.1.zip
+      LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
       ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
-      ONEAPI_INSTALLER_VERSION: "2026.1"
+      ONEAPI_INSTALLER_VERSION: "2025.3.3"
 
     steps:
       - name: Clone
@@ -1369,10 +1107,9 @@ jobs:
           "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-x64-sycl
-          evict-old-files: 1d
 
       - name: Build
         id: cmake_build
@@ -1392,11 +1129,9 @@ jobs:
         run: |
           echo "cp oneAPI running time dll files in ${{ env.ONEAPI_ROOT }} to ./build/bin"
 
-          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.6.dll" ./build/bin
-          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.3.dll" ./build/bin
-          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_def.3.dll" ./build/bin
-          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_avx2.3.dll" ./build/bin
-          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.3.dll" ./build/bin
+          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.5.dll" ./build/bin
+          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.2.dll" ./build/bin
+          cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.2.dll" ./build/bin
 
           cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero.dll" ./build/bin
           cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/ur_adapter_level_zero_v2.dll" ./build/bin
@@ -1411,11 +1146,13 @@ jobs:
             echo "Level Zero loader DLL not found in oneAPI or SDK; relying on system driver/runtime"
           fi
 
-          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl9.dll" ./build/bin
+          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl8.dll" ./build/bin
           cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/svml_dispmd.dll" ./build/bin
           cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libmmd.dll" ./build/bin
           cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libiomp5md.dll" ./build/bin
           cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl-ls.exe" ./build/bin
+          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-fallback-bfloat16.spv" ./build/bin
+          cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libsycl-native-bfloat16.spv" ./build/bin
 
           cp "${{ env.ONEAPI_ROOT }}/dnnl/latest/bin/dnnl.dll" ./build/bin
           cp "${{ env.ONEAPI_ROOT }}/tbb/latest/bin/tbb12.dll" ./build/bin
@@ -1455,8 +1192,8 @@ jobs:
 
     env:
       ONEAPI_ROOT: /opt/intel/oneapi/
-      ONEAPI_INSTALLER_VERSION: "2026.1"
-      LEVEL_ZERO_VERSION: "1.33.1"
+      ONEAPI_INSTALLER_VERSION: "2025.3.3"
+      LEVEL_ZERO_VERSION: "1.28.2"
       LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
 
     steps:
@@ -1470,16 +1207,16 @@ jobs:
         shell: bash
         run: |
           cd /tmp
-          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/5996e26b-f48a-42b1-8db0-b002ad0bd8d7/intel-oneapi-toolkit-2026.1.1.33_offline.sh -O intel-oneapi-toolkit_offline.sh
-          sudo bash intel-oneapi-toolkit_offline.sh -s -a --silent --eula accept
+          wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
+          sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
 
       - name: Install Level Zero SDK
         shell: bash
         run: |
           cd /tmp
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze1_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze1.deb
-          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/libze-dev_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O libze-dev.deb
-          sudo apt-get install -y ./libze1.deb ./libze-dev.deb
+          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
+          wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
+          sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
 
       - name: Download UI build
         uses: actions/download-artifact@v7
@@ -1488,10 +1225,9 @@ jobs:
           path: tools/ui/dist
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-ubuntu-24.04-sycl-${{ matrix.build }}
-          evict-old-files: 1d
 
       - name: Build
         id: cmake_build
@@ -1543,8 +1279,8 @@ jobs:
     strategy:
       matrix:
         include:
-          - ROCM_VERSION: "10.0.0"
-            gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
+          - ROCM_VERSION: "7.14.0"
+            gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
             build: 'x64'
 
     steps:
@@ -1566,7 +1302,7 @@ jobs:
           tool-cache: true
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
           evict-old-files: 1d
@@ -1597,7 +1333,7 @@ jobs:
           # libraries = HIP runtime and CMake configs needed for linking
           # devel = compilers, headers, static libs
           python -m pip install --upgrade pip
-          python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
+          python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
 
           # Get ROCm installation paths using the rocm-sdk CLI tool
           ROCM_PATH=$(rocm-sdk path --root)
@@ -1836,12 +1572,9 @@ jobs:
       - ubuntu-24-rocm
       - ubuntu-cpu
       - ubuntu-vulkan
-      - ubuntu-cuda
       - ubuntu-24-openvino
       - ubuntu-24-sycl
       - android-arm64
-      - android-arm64-snapdragon
-      - linux-arm64-snapdragon
       - macos-cpu
       - ios-xcode
       #- openEuler-cann
@@ -1970,30 +1703,25 @@ jobs:
             - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
             - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
             - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
-            - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
-            - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz)
-            - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz)
-            - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
+            - [Ubuntu x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.14-x64.tar.gz)
             - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)
-            - [Linux arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-linux-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md)
 
             **Android:**
             - [Android arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64.tar.gz)
-            - [Android arm64 (Snapdragon: CPU, Adreno GPU, Hexagon NPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-android-arm64-snapdragon.tar.gz) - [setup guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/README.md)
 
             **Windows:**
             - [Windows x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-x64.zip)
             - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
             - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
             - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
-            - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
-            - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
+            - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip)
+            - [Windows arm64 (CUDA 13) (preview)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
             - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
             - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
             - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
-            - [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip)
+            - [Windows x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-7.14-x64.zip)
 
             **openEuler:**
             - [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705)

.github/workflows/server-sanitize.yml

@@ -32,8 +32,6 @@ on:
     ]
 
 env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
   LLAMA_ARG_LOG_COLORS: 1
   LLAMA_ARG_LOG_PREFIX: 1
   LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -45,7 +43,7 @@ concurrency:
 
 jobs:
   server:
-    runs-on: hf-jobs-cpu-performance
+    runs-on: [self-hosted, CPU, Linux, llama-server]
 
     strategy:
       matrix:
@@ -54,6 +52,20 @@ jobs:
       fail-fast: false
 
     steps:
+      #- name: Dependencies
+      #  id: depends
+      #  run: |
+      #    sudo apt-get update
+      #    sudo apt-get -y install \
+      #      build-essential \
+      #      xxd \
+      #      git \
+      #      cmake \
+      #      curl \
+      #      wget \
+      #      language-pack-en \
+      #      libssl-dev
+
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
@@ -61,24 +73,6 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential cmake python3-full
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: server-sanitize-${{ matrix.sanitizer }}
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
@@ -93,17 +87,9 @@ jobs:
             -DLLAMA_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }}
           cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: server-sanitize-${{ matrix.sanitizer }}
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+      - name: Python setup
+        id: setup_python
+        uses: actions/setup-python@v7
 
       - name: Install Python dependencies
         run: |
@@ -116,7 +102,8 @@ jobs:
         run: |
           source .venv/bin/activate
           cd tools/server/tests
-          PYTEST_WORKERS=4 ./tests.sh
+          export ${{ matrix.extra_args }}
+          ./tests.sh
 
       - name: Slow tests
         id: server_integration_tests_slow
@@ -124,4 +111,5 @@ jobs:
         run: |
           source .venv/bin/activate
           cd tools/server/tests
-          PYTEST_WORKERS=4 SLOW_TESTS=1 ./tests.sh
+          export ${{ matrix.extra_args }}
+          SLOW_TESTS=1 ./tests.sh

.github/workflows/server-self-hosted.yml

@@ -72,7 +72,7 @@ jobs:
         run: |
           cd tools/server/tests
           source venv/bin/activate
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx1, backend-sampling)
         id: server_integration_tests_backend_sampling
@@ -81,7 +81,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2)
         id: server_integration_tests_gpu2
@@ -90,7 +90,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_METAL_DEVICES=2
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2, backend-sampling)
         id: server_integration_tests_gpu2_backend_sampling
@@ -99,10 +99,10 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
   server-cuda:
-    runs-on: "hf-jobs-t4-small:cuda13"
+    runs-on: [self-hosted, llama-server, Linux, NVIDIA]
 
     steps:
       - name: Clone
@@ -112,42 +112,12 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y cmake libssl-dev python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: self-hosted-server-cuda
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
-          cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+          cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON
           cmake --build build --config Release -j $(nproc) --target llama-server
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: self-hosted-server-cuda
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Python setup
         id: setup_python
         run: |
@@ -162,7 +132,7 @@ jobs:
         run: |
           cd tools/server/tests
           source venv/bin/activate
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx1, backend-sampling)
         id: server_integration_tests_backend_sampling
@@ -171,7 +141,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2)
         id: server_integration_tests_gpu2
@@ -180,7 +150,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_CUDA_DEVICES=2
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2, backend-sampling)
         id: server_integration_tests_gpu2_backend_sampling
@@ -189,10 +159,10 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
   server-kleidiai:
-    runs-on: ah-ubuntu_24_04-c8g_8x
+    runs-on: ah-ubuntu_22_04-c8g_8x
 
     steps:
       - name: Clone
@@ -232,7 +202,7 @@ jobs:
       - name: Build
         id: cmake_build
         run: |
-          cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON -DLLAMA_FATAL_WARNINGS=ON
+          cmake -B build -DGGML_SCHED_NO_REALLOC=ON -DGGML_CPU_KLEIDIAI=ON
           cmake --build build --config Release -j $(nproc) --target llama-server
 
       - name: Python setup

.github/workflows/server.yml

@@ -80,19 +80,11 @@ jobs:
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: server-ubuntu-24.04-arm
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: server-ubuntu-24.04-arm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -101,18 +93,6 @@ jobs:
             -DGGML_SCHED_NO_REALLOC=ON
           cmake --build build --config Release -j $(nproc) --target llama-server
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: server-ubuntu-24.04-arm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Python setup
         id: setup_python
         uses: actions/setup-python@v6
@@ -148,6 +128,16 @@ jobs:
           export LLAMA_ARG_BACKEND_SAMPLING=1
           SLOW_TESTS=1 ./tests.sh
 
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: server-ubuntu-24.04-arm
+          older: 5m
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+
   windows:
     runs-on: windows-2025
 
@@ -160,7 +150,7 @@ jobs:
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: server-windows-2025-x64
           evict-old-files: 1d

.github/workflows/ui-build-self-hosted.yml

@@ -17,9 +17,8 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Install dependencies
         run: npm ci

.github/workflows/ui-build.yml

@@ -33,9 +33,8 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Install dependencies
         run: npm ci

.github/workflows/ui.yml

@@ -57,9 +57,8 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Download built UI artifacts
         uses: actions/download-artifact@v6
@@ -115,9 +114,8 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Install dependencies
         id: setup

.github/workflows/update-ops-docs.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
     update-ops-docs:
-        runs-on: ubuntu-slim
+        runs-on: [self-hosted, fast, ARM64]
 
         steps:
         - name: Checkout repository

.github/workflows/winget.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   update:
     name: Update Winget Package
-    runs-on: ubuntu-24.04  # previously ubuntu-latest
+    runs-on: ubuntu-latest
     if: github.repository_owner == 'ggml-org'
 
     steps:

.pi/gg/SYSTEM.md

@@ -6,8 +6,6 @@ General:
 - PR and commit titles format: `<module> : <title>`. Lookup recents for examples
 - Don't try to build or run the code unless you are explicitly asked to do so
 - Use the `gh` CLI tool when querying PRs, issues, or other GitHub resources
-- When [MODEL] is needed, first try to get it from the `PI_MODEL_NAME` env var before asking the user
-- Never read the `AGENTS.md` file
 
 Coding:
 - When in doubt, always refer to the CONTRIBUTING.md file of the project
@@ -22,9 +20,8 @@ Pull requests (PRs):
 - Don't explicitly wrap lines in the PR description (each paragraph and bullet is a single line)
 - When creating a pull request, look for the repository's PR template and follow it
 - For the AI usage disclosure section, write "YES. pi:llama.cpp/[MODEL]"
-- If `PI_MODEL_NAME` env var is not set, ask the user to tell you what model was used and write it in place of [MODEL]
+- Ask the user to tell you what model was used and write it in place of [MODEL]
 - Always create the pull requests in draft mode
-- Never reply to review comments or post comments on issues/PRs without explicit permission from the user
 
 Commits:
 - On every commit that you make, include a "Assisted-by: pi:llama.cpp/[MODEL]" tag

AUTHORS

@@ -1,4 +1,4 @@
-# date: Fri Sep  4 10:06:46 EEST 2026
+# date: Tue Aug 18 14:32:43 EEST 2026
 # this file is auto-generated by scripts/gen-authors.sh
 
 Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
@@ -46,7 +46,6 @@ Abhijit Ramesh <abhijitramesh2k@gmail.com>
 abhijitb11 <113058133+abhijitb11@users.noreply.github.com>
 Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
 Abhinay Krishna <abhinaykrishna60@gmail.com>
-Abhiram <78226909+geckguy@users.noreply.github.com>
 Abhishek Gopinath K <31348521+overtunned@users.noreply.github.com>
 abotsis <github@bots.is>
 Abraham Gonzalez <theabecaster0@gmail.com>
@@ -88,7 +87,6 @@ akleine <alb.kleine@gmx.de>
 Al G <toasting@gmail.com>
 Al Mochkin <14274697+amochkin@users.noreply.github.com>
 Alan Gray <agray3@users.noreply.github.com>
-Alan Tseng <alanhc.tseng1999@gmail.com>
 Alawode Oluwandabira <dabiraalawode@yahoo.com>
 Albert Jin <albert.jin@gmail.com>
 Alberto <57916483+albbus-stack@users.noreply.github.com>
@@ -138,9 +136,7 @@ alonfaraj <alonfaraj@gmail.com>
 AlpinDale <52078762+AlpinDale@users.noreply.github.com>
 alwqx <kenan3015@gmail.com>
 Aman <amangupta052@gmail.com>
-Aman Chadha(IVIXMMI) <79802170+ac-mmi@users.noreply.github.com>
 Aman Gupta <amangupta052@gmail.com>
-Aman Karki <itsamankarki@gmail.com>
 amd-dwang <dong.wang@amd.com>
 amd-lalithnc <lalithnc@amd.com>
 Amir <amir_zia@outlook.com>
@@ -191,7 +187,6 @@ Anton Mitkov <anton.mitkov@codeplay.com>
 Antonis Makropoulos <benuix@gmail.com>
 Anudit Nagar <nagaranudit@gmail.com>
 Anuj Attri <anujattri01@gmail.com>
-anujj <ajalota@nvidia.com>
 anzz1 <anzz1@live.com>
 Aparna M P <aparmp@qti.qualcomm.com>
 Aparna M P <quic_aparmp@quicinc.com>
@@ -201,7 +196,6 @@ arch-btw <57669023+arch-btw@users.noreply.github.com>
 arcrank <arcrank@gmail.com>
 ardfork <134447697+ardfork@users.noreply.github.com>
 Arik Poznanski <arikpoz@users.noreply.github.com>
-Aritro Bandyopadhyay <71339004+AriBandyo@users.noreply.github.com>
 arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
 Armen Kaleshian <kriation@users.noreply.github.com>
 Arsen Arutunan <58118221+limloop@users.noreply.github.com>
@@ -236,7 +230,6 @@ bandoti <141645996+bandoti@users.noreply.github.com>
 Bar Haim <barvhaim@gmail.com>
 BarfingLemurs <128182951+BarfingLemurs@users.noreply.github.com>
 Bart Louwers <bart.louwers@gmail.com>
-Bartosz Taudul <wolf@nereid.pl>
 Bartowski <3266127+bartowski1182@users.noreply.github.com>
 Bartowski <ckealty1182@gmail.com>
 Bas Nijholt <basnijholt@gmail.com>
@@ -284,7 +277,6 @@ Bono Lv <lvscar@users.noreply.github.com>
 Borislav Stanimirov <b.stanimirov@abv.bg>
 Borislav Stanimirov <b@ibob.bg>
 Bowen Han <fancycode@gmail.com>
-Brad Smith <1472326+infinitewarp@users.noreply.github.com>
 Branden Butler <bwtbutler@hotmail.com>
 Brandon Squizzato <35474886+bsquizz@users.noreply.github.com>
 Brian <mofosyne@gmail.com>
@@ -295,7 +287,6 @@ Bryan Honof <bryanhonof@gmail.com>
 bryanSwk <93190252+bryanSwk@users.noreply.github.com>
 bsilvereagle <bsilvereagle@users.noreply.github.com>
 bssrdf <merlintiger@hotmail.com>
-Buğra Özgürsoy <13810383+ozgursoy@users.noreply.github.com>
 byte-6174 <88070277+byte-6174@users.noreply.github.com>
 Caleb DeLeeuw <143902425+SolshineCode@users.noreply.github.com>
 Calvin Laurenson <calvin@laurenson.dev>
@@ -335,7 +326,6 @@ Chenguang Li <757486878@qq.com>
 Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
 Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
 chiranko <96988916+chiranko@users.noreply.github.com>
-Chris Danis <cdanis@gmail.com>
 Chris Elrod <elrodc@gmail.com>
 Chris Kuehl <ckuehl@ckuehl.me>
 Chris Lee <clee@mg8.org>
@@ -366,7 +356,6 @@ clyang <clyang@clyang.net>
 cmdr2 <secondary.cmdr2@gmail.com>
 cmdr2 <shashank.shekhar.global@gmail.com>
 cocktailpeanut <121128867+cocktailpeanut@users.noreply.github.com>
-codemonkey <441345965@qq.com>
 codezjx <code.zjx@gmail.com>
 coezbek <c.oezbek@gmail.com>
 comex <comexk@gmail.com>
@@ -378,8 +367,6 @@ Copilot <198982749+Copilot@users.noreply.github.com>
 Corentin REGAL <corentin.regal@gmail.com>
 cphlipot <9103367+cphlipot@users.noreply.github.com>
 cpumaxx <163466046+cpumaxx@users.noreply.github.com>
-cqderek <cqderek@gmail.com>
-cqderek <cqiang@qti.qualcomm.com>
 crasm <crasm@git.vczf.net>
 crasm <crasm@git.vczf.us>
 crat0z <11581854+crat0z@users.noreply.github.com>
@@ -440,7 +427,6 @@ DavidKorczynski <david@adalogics.com>
 davidrhodus <david@vacovideo.com>
 Dawid Potocki <github@dawidpotocki.com>
 Dawid Wysocki <62249621+TortillaZHawaii@users.noreply.github.com>
-Daya Adianto <addianto@users.noreply.github.com>
 ddh0 <chemist-mulches-39@icloud.com>
 ddh0 <dylanhalladay02@icloud.com>
 ddpasa <112642920+ddpasa@users.noreply.github.com>
@@ -477,7 +463,6 @@ Dmytro Romanov <casteldazur@gmail.com>
 Dobri Danchev <12420863+danchev@users.noreply.github.com>
 DocShotgun <126566557+DocShotgun@users.noreply.github.com>
 Doctor Shotgun <126566557+DocShotgun@users.noreply.github.com>
-Dominik Pantaleoni <95251853+dpantaleoni@users.noreply.github.com>
 Don Mahurin <dmahurin@users.noreply.github.com>
 Dong Won Kim <63934649+ddwkim@users.noreply.github.com>
 Donghyeon Jeong <54725479+djeong20@users.noreply.github.com>
@@ -519,7 +504,6 @@ Emmanuel Ferdman <emmanuelferdman@gmail.com>
 Emreerdog <34742675+Emreerdog@users.noreply.github.com>
 Engininja2 <139037756+Engininja2@users.noreply.github.com>
 Equim <sayaka@ekyu.moe>
-Eric A Stalee <87948564+Eric-A-Stalee@users.noreply.github.com>
 Eric Curtin <ecurtin@redhat.com>
 Eric Curtin <eric.curtin@docker.com>
 Eric Curtin <ericcurtin17@gmail.com>
@@ -535,7 +519,6 @@ Esko Toivonen <eskot98@gmail.com>
 Ethan Turner <eturner64@gmail.com>
 Ettore Di Giacinto <mudler@users.noreply.github.com>
 EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
-Eurekatic <eurekatic@eurekatic.eu>
 Evan Huus <eapache@gmail.com>
 Evan Jones <evan.q.jones@gmail.com>
 Evan Miller <emmiller@gmail.com>
@@ -694,7 +677,6 @@ HimariO <dsfhe49854@gmail.com>
 hipudding <huafengchun@gmail.com>
 Hitesh Chopra <34310832+hiteshchopra11@users.noreply.github.com>
 hksdpc255 <43977088+hksdpc255@users.noreply.github.com>
-hmirin <hmirin@users.noreply.github.com>
 hmscider <201289679+hmscider@users.noreply.github.com>
 Hoang Nguyen <hugo53@users.noreply.github.com>
 hoangmit <hoangmit@users.noreply.github.com>
@@ -719,7 +701,6 @@ Huawei Lin <huaweilin.cs@gmail.com>
 Hugo <hugo@whynothugo.nl>
 Hugo Roussel <hugo.rous@gmail.com>
 Huifeng Ou <79071290+ho2103@users.noreply.github.com>
-HumerousGorgon <31957201+HumerousGorgon@users.noreply.github.com>
 hutli <6594598+hutli@users.noreply.github.com>
 hutli <hutli@hutli.hu>
 hutli <jensstaermose@hotmail.com>
@@ -757,15 +738,12 @@ intelmatt <61025942+intelmatt@users.noreply.github.com>
 iohub <rickyang.pro@gmail.com>
 Ionoclast Laboratories <brigham@ionoclast.com>
 iron <lizhenneng@gmail.com>
-Isaac <34376531+init-22@users.noreply.github.com>
 Isaac McFadyen <isaac@imcf.me>
 IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
 Ishaan Gandhi <Ishaangandhi@gmail.com>
 iSma <ismail.senhaji@gmail.com>
 Ismail <115064057+AlrIsmail@users.noreply.github.com>
 issixx <46835150+issixx@users.noreply.github.com>
-itsnotoger <19309683+itsnotoger@users.noreply.github.com>
-itterative <190138728+itterative@users.noreply.github.com>
 Ivan <nekotekina@gmail.com>
 Ivan Chikish <nekotekina@gmail.com>
 Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
@@ -790,7 +768,6 @@ Jakkala Mahesh <155058658+MaheshJakkala@users.noreply.github.com>
 Jakub N <jakubniemczyk97@gmail.com>
 JamePeng <jame_peng@sina.com>
 James A Capozzoli <157492257+jac-jim@users.noreply.github.com>
-James Francis <6763899+JamesFranc@users.noreply.github.com>
 James O'Leary <65884233+jpohhhh@users.noreply.github.com>
 James Reynolds <magnusviri@users.noreply.github.com>
 jameswu2014 <545426914@qq.com>
@@ -821,7 +798,6 @@ Jed Fox <git@jedfox.com>
 Jeff Bolz <jbolz@nvidia.com>
 Jeffrey Morgan <jmorganca@gmail.com>
 Jeffrey Quesnelle <emozilla@nousresearch.com>
-Jeremie Miller <jeremie.miller@gmail.com>
 Jeremy Demeule <jdemeule@users.noreply.github.com>
 Jeremy Rand <244188+JeremyRand@users.noreply.github.com>
 Jeroen Mostert <jeroen.mostert@cm.com>
@@ -833,7 +809,6 @@ Jesse Jojo Johnson <williamsaintgeorge@gmail.com>
 Jesse LaRose <jesse@taey.ai>
 Jesse Posner <jesse.posner@gmail.com>
 Jesus Talavera <145992175+jesus-talavera-ibm@users.noreply.github.com>
-Jetson Tan <tanzongyouyi@outlook.com>
 Jett Janiak <jettjaniak@gmail.com>
 Jeximo <jeximo@gmail.com>
 JFLFY2255 <JFLFY2255@163.com>
@@ -850,7 +825,6 @@ Jie Fu (傅杰) <jiefu@tencent.com>
 jiez <373447296@qq.com>
 Jillis ter Hove <j.terhove@gmail.com>
 Jim Wu <jimw567@users.noreply.github.com>
-Jingxin (Philip) Li <philipaslee@gmail.com>
 Jinwoo Jeong <33892306+williamjeong2@users.noreply.github.com>
 Jinyang He <hejinyang@loongson.cn>
 jinzihao <jinzihao1996@gmail.com>
@@ -876,13 +850,11 @@ John Balis <phobossystems@gmail.com>
 John Bean <113509988+johnbean393@users.noreply.github.com>
 John Eismeier <42679190+jeis4wpi@users.noreply.github.com>
 John Smith <67539080+kingsidelee@users.noreply.github.com>
-John-Henry Lim <42513874+Interpause@users.noreply.github.com>
 Johnathan Craig Maudlin <13183098+jcmdln@users.noreply.github.com>
 JohnnyB <jboero@users.noreply.github.com>
 johnson442 <56517414+johnson442@users.noreply.github.com>
 jojorne <jojorne@users.noreply.github.com>
 jon-chuang <9093549+jon-chuang@users.noreply.github.com>
-Jonas J <111707981+John-194@users.noreply.github.com>
 Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
 Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
 Jonathan <47618606+jbuchananr@users.noreply.github.com>
@@ -952,7 +924,6 @@ Karsten Weiss <knweiss@gmail.com>
 Karthick <j.karthic2004@gmail.com>
 Karthik Kumar Viswanathan <195178+guilt@users.noreply.github.com>
 Karthik Sethuraman <k.seth1993@gmail.com>
-Kartik Gulia <kgulia@nvidia.com>
 Kartik Sirohi <99896785+sirohikartik@users.noreply.github.com>
 Kashif Rasul <kashif.rasul@gmail.com>
 KASR <karim.asrih@gmail.com>
@@ -960,7 +931,6 @@ Kasumi <90275229+kasumi-1@users.noreply.github.com>
 Katostrofik <georgiopapairo@gmail.com>
 katsu560 <118887472+katsu560@users.noreply.github.com>
 Kawrakow <48489457+ikawrakow@users.noreply.github.com>
-kbenkhaled <khalilbenkhaled01@gmail.com>
 kchro3 <62481661+kchro3@users.noreply.github.com>
 kdkd <2569413+kdkd@users.noreply.github.com>
 Keiichi Tabata <keiichi.tabata@outlook.com>
@@ -969,7 +939,6 @@ Kenvix ⭐ <kenvixzure@live.com>
 Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com>
 Kevin Gibbons <bakkot@gmail.com>
 Kevin Hannon <kehannon@redhat.com>
-Kevin Hopper <93635715+kh0pper@users.noreply.github.com>
 Kevin Ji <1146876+kevinji@users.noreply.github.com>
 Kevin Kwok <antimatter15@gmail.com>
 Kevin Liu <4396kevinliu@gmail.com>
@@ -995,14 +964,12 @@ Konstantin Herud <konstantin.herud@denkbares.com>
 Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
 Krishna Sridhar <99914379+srikris-sridhar@users.noreply.github.com>
 krystiancha <krystian@krystianch.com>
-krzsztf <krzysztof@witkowscy.org>
 kubawoo <k-wach@o2.pl>
 kumaal <44551860+kumaal@users.noreply.github.com>
 kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com>
 kunnis <kunnis@users.noreply.github.com>
 Kunshang Ji <kunshang.ji@intel.com>
 kuronekosaiko <EvanChanJ@163.com>
-kurquhar <kurquhar@qti.qualcomm.com>
 Kusha Gharahi <3326002+kushagharahi@users.noreply.github.com>
 kustaaya <58045274+kustaaya@users.noreply.github.com>
 kuvaus <22169537+kuvaus@users.noreply.github.com>
@@ -1014,7 +981,6 @@ Kyle Liang <liangmanlai@gmail.com>
 Kyle Mistele <kyle@mistele.com>
 KyleHagy <59183061+KyleHagy@users.noreply.github.com>
 Kylin <56434533+KyL0N@users.noreply.github.com>
-Kyozzz <1147385157@qq.com>
 l-austenfeld <53152202+l-austenfeld@users.noreply.github.com>
 l3utterfly <gc.pthzfoldr@gmail.com>
 l8bloom <l8bloomapi@gmail.com>
@@ -1026,7 +992,6 @@ Lars Sonchocky-Helldorf <lars.sonchocky-helldorf@hamburg.de>
 las7 <98077186+las7@users.noreply.github.com>
 Lasse Lauwerys <65569591+Iemand005@users.noreply.github.com>
 Laura <Tijntje_7@msn.com>
-Laurent Zuijdwijk <laurent.zuijdwijk@gmail.com>
 Law Po Ying <30721578+yingying0906@users.noreply.github.com>
 lcy <lcy0321@users.noreply.github.com>
 ldwang <ftgreat@163.com>
@@ -1074,8 +1039,6 @@ Ludovic Henry <git@ludovic.dev>
 Ludovic Henry <ludovic@rivosinc.com>
 Lukas Straub <lukasstraub2@web.de>
 Łukasz Ślusarczyk <112692748+lslusarczyk@users.noreply.github.com>
-Lukasz Stolcman <4583553+lstolcman@users.noreply.github.com>
-LunalFresh <165352784+LunalFresh@users.noreply.github.com>
 Luo Tian <lt@basecity.com>
 luoyu-intel <yu.luo@intel.com>
 luyhcsu <110711054+luyhcsu@users.noreply.github.com>
@@ -1091,7 +1054,6 @@ Maarten ter Huurne <maarten@treewalker.org>
 Maciej Lisowski <39798354+MaciejDromin@users.noreply.github.com>
 Mack Straight <eiz@users.noreply.github.com>
 maddes8cht <55592906+maddes8cht@users.noreply.github.com>
-Mads Marquart <mads@marquart.dk>
 Maël Kerbiriou <m431.kerbiriou@gmail.com>
 MaggotHATE <clay1326@gmail.com>
 MagicExists <106458387+gugugiyu@users.noreply.github.com>
@@ -1253,8 +1215,6 @@ Naco Siren <naco-siren@users.noreply.github.com>
 Nam D. Tran <42194884+namtranase@users.noreply.github.com>
 nanahi <130121847+na-na-hi@users.noreply.github.com>
 Nathan Epstein <nate2@umbc.edu>
-Nathan Wilson <67372905+Nathanw1014@users.noreply.github.com>
-Nathanw1014 <67372905+Nathanw1014@users.noreply.github.com>
 Natsu <chino@hotococoa.moe>
 Nauful Shaikh <nauful@gmail.com>
 NawafAlansari <72708095+NawafAlansari@users.noreply.github.com>
@@ -1277,7 +1237,6 @@ niansa/tuxifan <tuxifan@posteo.de>
 Nicholai Tukanov <nicholaitukanov@gmail.com>
 Nicholas Sparks <157740354+nisparks@users.noreply.github.com>
 Nick <0x0b4ac@gmail.com>
-Nick Farrell <nick.farrell@aiven.io>
 nick huang <nickhuang99@hotmail.com>
 Nick Lafleur <55208706+nicklafleur@users.noreply.github.com>
 Nick Towle <ntowle@gmail.com>
@@ -1300,7 +1259,6 @@ NikolaiLyssogor <59844691+NikolaiLyssogor@users.noreply.github.com>
 Nikolaos Pothitos <pothitos@di.uoa.gr>
 Nikolas <127742645+nneubacher@users.noreply.github.com>
 Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
-Nils Gladitz <nilsgladitz@gmail.com>
 Nindaleth <Nindaleth@users.noreply.github.com>
 ningshanwutuobang <ningshanwutuobang@gmail.com>
 Noah <99681487+NoahOksuz@users.noreply.github.com>
@@ -1397,7 +1355,6 @@ Pop Flamingo <trevor.annedenise@icloud.com>
 postmasters <namnguyen@google.com>
 Pouya <PooyaGhahramanian@Gmail.com>
 pqnet <119850+pqnet@users.noreply.github.com>
-Prabhsimran Singh <pskrunner14@gmail.com>
 Prabod <prabod@maincode.com>
 Prajwal B Mehendarkar <prajwal.b.mehendarkar@ibm.com>
 Pranav Dhinakar <pdhinaka@qti.qualcomm.com>
@@ -1421,7 +1378,6 @@ qouoq <qouoq@fastmail.com>
 Qu Zongfu <43257352+yancaoweidaode@users.noreply.github.com>
 quei <56998528+quei4r@users.noreply.github.com>
 Quentin Bramas <quentin.bramas@gmail.com>
-QuintinShaw <github@xyt.email>
 QuintinShaw <yx6f20@soton.ac.uk>
 qunash <anzoria@gmail.com>
 quyentonndbs <raynaedgar8677@outlook.com>
@@ -1506,7 +1462,6 @@ robertomeroni <150194833+robertomeroni@users.noreply.github.com>
 Robey Holderith <robey@flaminglunchbox.net>
 Robin Davidsson <40024429+R-Dson@users.noreply.github.com>
 Robyn <robyngraf@users.noreply.github.com>
-Rock Chen <rockchen.tw@gmail.com>
 Rőczey Barnabás <31726601+An0nie@users.noreply.github.com>
 RodriMora <bullerwins@gmail.com>
 Roger Chen <chenrui@gmail.com>
@@ -1544,21 +1499,17 @@ runfuture <runfuture@users.noreply.github.com>
 RunningLeon <maningsheng@sensetime.com>
 RunningLeon <mnsheng@yeah.net>
 Russyyds <161207317+Russyyds@users.noreply.github.com>
-Ryan C <ryan5rdx@users.noreply.github.com>
 Ryan Goulden <percontation@gmail.com>
 Ryan Landay <rlanday@gmail.com>
 Ryan Mangeno <160974989+ryan-mangeno@users.noreply.github.com>
 Ryder Wishart <ryderwishart@gmail.com>
 Ryuei <louixs@users.noreply.github.com>
 s-goto-11 <206795233+s-goto-11@users.noreply.github.com>
-s0mecode <213953308+s0mecode@users.noreply.github.com>
 s8322 <s0527684199@gmail.com>
-Saad Ali <NIXKnight@users.noreply.github.com>
 Saba Fallah <10401143+sfallah@users.noreply.github.com>
 Saba Fallah <sabafallah@gmail.com>
 Sachin Desai <smdesai@gmail.com>
 Sachin Sharma <sachin@zettabolt.com>
-Safi Ullah <safiullah.3915@gmail.com>
 safranowith <bsh155762@gmail.com>
 SakuraUmi <yukinon244@gmail.com>
 Salvador E. Tropea <stropea@inti.gob.ar>
@@ -1601,7 +1552,6 @@ Sergey Alirzaev <l29ah@riseup.net>
 Sergey Alirzaev <zl29ah@gmail.com>
 Sergey Fedorov <vital.had@gmail.com>
 Sergey Malinin <sergmalinin@gmail.com>
-Sergey Sklyarov <sergey.sklyarov@gmail.com>
 Sergio López <slp@redhat.com>
 Sergio López <slp@sinrega.org>
 Sergiu <8598216+mzsergiu@users.noreply.github.com>
@@ -1632,13 +1582,11 @@ Shawn Gu <shawngu@qti.qualcomm.com>
 Shawn yang <137684499+Yangxiaoz@users.noreply.github.com>
 Shelby Jenkins <47464908+ShelbyJenkins@users.noreply.github.com>
 Sheldon Robinson <sheldon.robinson@live.com>
-Shenghan Yang <ysharke@sjtu.edu.cn>
 shibe2 <shibe@tuta.io>
 Shijie <821898965@qq.com>
 Shin-myoung-serp <relent95@naver.com>
 Shintarou Okada <kokuzen@gmail.com>
 shivamkumard-ctrl <shivamkumard@nvidia.com>
-Shobhit <sobhit.me@gmail.com>
 Shouyu <65317431+joeldushouyu@users.noreply.github.com>
 Shouzheng Liu <61452103+lshzh-ww@users.noreply.github.com>
 Shouzheng Liu <lshzh.hi@gmail.com>
@@ -1659,7 +1607,6 @@ Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
 Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
 simevo <github@simevo.com>
 Simon Redman <simon@ergotech.com>
-Simon Teixidor <simon@flaskpost.me>
 Simon Willison <swillison@gmail.com>
 simon886212 <37953122+simon886212@users.noreply.github.com>
 Simranjeet Singh <105192966+simrnsingh@users.noreply.github.com>
@@ -1716,7 +1663,6 @@ stevenkuang <stevenkuang@tencent.com>
 Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
 StrangeBytesDev <141275258+StrangeBytesDev@users.noreply.github.com>
 strawberrymelonpanda <152940198+strawberrymelonpanda@users.noreply.github.com>
-Strongtut <Strongtut@users.noreply.github.com>
 Suaj Carrot <72162667+SuajCarrot@users.noreply.github.com>
 sudhiarm <sudhi.sathyavathy@arm.com>
 Sukriti Sharma <Ssukriti@users.noreply.github.com>
@@ -1741,7 +1687,6 @@ Tamar <Tamar0812@outlook.co.il>
 tamarPal <tamarp3385@gmail.com>
 Tameem <113388789+AhmadTameem@users.noreply.github.com>
 Tamotsu Takahashi <ttakah+github@gmail.com>
-Tanner Bruhn <66120666+tannerbruhn@users.noreply.github.com>
 tarcey <cey.tarik@gmail.com>
 Tarek Dakhran <t.dakhran@gmail.com>
 Tarek Dakhran <tarek@liquid.ai>
@@ -1751,7 +1696,6 @@ Taylor <quantumtraveling@gmail.com>
 tc-mb <157115220+tc-mb@users.noreply.github.com>
 TecJesh <qdvm5gl@163.com>
 Tei Home <taiteitonghome@proton.me>
-Tekin Ertekin <tekin.ertekin@gmail.com>
 Tekin Ertekin <tekinertekin@gmail.com>
 tempstudio <49735574+tempstudio@users.noreply.github.com>
 teo <TeoZosa@users.noreply.github.com>
@@ -1793,7 +1737,6 @@ Ting Lou <louting@189.cn>
 Ting Lou <ting.lou@gmail.com>
 Ting Sun <suntcrick@gmail.com>
 Titaniumtown <titaniumtown@proton.me>
-Tiwei Bie <tiwei.btw@antgroup.com>
 tjohnman <tjohnman@users.noreply.github.com>
 Tobias Lütke <tobi@shopify.com>
 Toby <25832191+aetherbird@users.noreply.github.com>
@@ -1870,7 +1813,6 @@ Vishal Agarwal <vishalagarwal.jss@gmail.com>
 Vishal Singh <vishal@zettabolt.com>
 Vitali Lovich <vlovich+github@gmail.com>
 Vivian <vynride@gmail.com>
-vk <89937361+itsvedantkumar@users.noreply.github.com>
 Vlad <spitfireage@gmail.com>
 Vladimir <bogdad@gmail.com>
 Vladimir Malyutin <first-leon@yandex.ru>
@@ -1955,7 +1897,6 @@ Yaiko <elyaiko@hotmail.com>
 Yakine Tahtah <96926916+ReinforcedKnowledge@users.noreply.github.com>
 YangLe <smilingpoplar@gmail.com>
 yangli2 <yangli2@gmail.com>
-Yaniss Amazouz <yaniss91600@gmail.com>
 Yann Follet <131855179+YannFollet@users.noreply.github.com>
 Yanzhao Wang <yanzhaow@qti.qualcomm.com>
 Yarden Tal <yardent@qti.qualcomm.com>

CMakeLists.txt

@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)
 
 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 5)
+set(LLAMA_VERSION_MINOR 3)
 set(LLAMA_VERSION_PATCH 0)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
 
@@ -197,16 +197,6 @@ llama_option_depr(WARNING     LLAMA_CURL)
 include("cmake/license.cmake")
 license_add_file("llama.cpp" "LICENSE")
 
-#
-# compile options
-#
-
-# clang stores the modification time of the precompiled header sources inside the
-# header and rejects it when they differ, so the timestamp is left out of it
-add_compile_options(
-    "$<$<COMPILE_LANG_AND_ID:C,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>"
-    "$<$<COMPILE_LANG_AND_ID:CXX,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>")
-
 #
 # 3rd-party
 #

CODEOWNERS

@@ -57,7 +57,6 @@
 /ggml/src/ggml-cann/                    @ggml-org/ggml-cann
 /ggml/src/ggml-common.h                 @ggerganov
 /ggml/src/ggml-cpu/                     @ggerganov
-/ggml/src/ggml-cpu/tiled/               @jbooth @bartowski1182
 /ggml/src/ggml-cpu/spacemit/            @alex-spacemit
 /ggml/src/ggml-cuda/                    @ggml-org/ggml-cuda
 /ggml/src/ggml-cuda/vendors/hip.h       @IMbackK
@@ -77,7 +76,7 @@
 /ggml/src/ggml-vulkan/                  @ggml-org/ggml-vulkan
 /ggml/src/ggml-webgpu/                  @ggml-org/ggml-webgpu
 /ggml/src/ggml-zdnn/                    @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
-/ggml/src/ggml-zendnn/                  @avinashcpandey @Jiten1parmar
+/ggml/src/ggml-zendnn/                  @avinashcpandey @Jiten1parmar @z-vishal
 /ggml/src/ggml.c                        @ggerganov
 /ggml/src/ggml.cpp                      @ggerganov
 /ggml/src/gguf.cpp                      @JohannesGaessler @Green-Sky
@@ -97,6 +96,7 @@
 /src/models/                            @CISC
 /tests/                                 @ggerganov
 /tests/test-chat.*                      @pwilkin
+/tests/test-llama-archs.cpp             @JohannesGaessler
 /tools/batched-bench/                   @ggerganov
 /tools/cli/                             @ngxson
 /tools/completion/                      @ggerganov

CONTRIBUTING.md

@@ -20,8 +20,8 @@ If AI is used to generate any portion of the code, contributors must adhere to t
 
 1. Explicitly disclose the manner in which AI was employed.
 2. Check for an existing PR addressing the same change; if one exists, comment there to work with its author instead of opening a duplicate.
-3. Perform a comprehensive manual review prior to submitting the pull request. A proper code review usually takes something like one hour per 200-400 LOC and you should be spending **at least that much time on code review alone**.
-4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
+3. Perform a comprehensive manual review prior to submitting the pull request.
+4. Be prepared to explain every line of code they submitted when asked about it by a maintainer.
 5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
 
 For more info, please refer to the [AGENTS.md](AGENTS.md) file.

README.md

@@ -13,7 +13,7 @@
 [![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
 [![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
 
-[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
+[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
 
 </div>
 
@@ -74,7 +74,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 | [CANN](docs/build.md#cann) | Ascend NPU |
 | [CUDA](docs/build.md#cuda) | Nvidia GPU |
 | [HIP](docs/build.md#hip) | AMD GPU |
-| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
+| [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon |
 | [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
 | [MUSA](docs/build.md#musa) | Moore Threads GPU |
 | [Metal](docs/build.md#metal-build) | Apple Silicon |

app/CMakeLists.txt

@@ -16,7 +16,7 @@ target_link_libraries(${TARGET} PRIVATE
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
 
 # Automatically add all files from the 'licenses' directory
-file(GLOB EXTRA_LICENSES "${PROJECT_SOURCE_DIR}/licenses/LICENSE-*")
+file(GLOB EXTRA_LICENSES "${CMAKE_SOURCE_DIR}/licenses/LICENSE-*")
 
 foreach(FILE_PATH ${EXTRA_LICENSES})
     get_filename_component(FILE_NAME "${FILE_PATH}" NAME)

app/llama.cpp

@@ -80,7 +80,7 @@ static const command cmds[] = {
 #undef UPDATE_HIDDEN
 
 static int version(int /*argc*/, char ** /*argv*/) {
-    llama_print_build_info(llama_version(), stdout);
+    llama_print_build_info(llama_version());
     return 0;
 }
 

build-xcframework.sh

@@ -18,7 +18,7 @@ LLAMA_BUILD_TESTS=OFF
 LLAMA_BUILD_SERVER=OFF
 LLAMA_BUILD_MTMD=ON
 GGML_METAL=ON
-GGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY:-ON}
+GGML_METAL_EMBED_LIBRARY=ON
 GGML_BLAS_DEFAULT=ON
 GGML_OPENMP=OFF
 
@@ -169,14 +169,6 @@ setup_framework_structure() {
     cp tools/mtmd/mtmd.h           ${header_path}
     cp tools/mtmd/mtmd-helper.h    ${header_path}
 
-    if [[ "$GGML_METAL_EMBED_LIBRARY" == "OFF" ]]; then
-        if [[ "$platform" == "macos" ]]; then
-            cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/Versions/A/Resources/
-        else
-            cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/
-        fi
-    fi
-
     # Create module map (common for all platforms)
     cat > ${module_path}module.modulemap << EOF
 framework module llama {
@@ -458,7 +450,6 @@ build_ios_sim() {
         -DIOS=ON \
         -DCMAKE_SYSTEM_NAME=iOS \
         -DCMAKE_OSX_SYSROOT=iphonesimulator \
-        -DGGML_METAL_TARGET_OS=ios \
         -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
@@ -476,7 +467,6 @@ build_ios_device() {
         -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
         -DCMAKE_SYSTEM_NAME=iOS \
         -DCMAKE_OSX_SYSROOT=iphoneos \
-        -DGGML_METAL_TARGET_OS=ios \
         -DCMAKE_OSX_ARCHITECTURES="arm64" \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
@@ -508,7 +498,6 @@ build_visionos() {
         -DCMAKE_OSX_ARCHITECTURES="arm64" \
         -DCMAKE_SYSTEM_NAME=visionOS \
         -DCMAKE_OSX_SYSROOT=xros \
-        -DGGML_METAL_TARGET_OS=xros \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
         -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
@@ -527,7 +516,6 @@ build_visionos_sim() {
         -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
         -DCMAKE_SYSTEM_NAME=visionOS \
         -DCMAKE_OSX_SYSROOT=xrsimulator \
-        -DGGML_METAL_TARGET_OS=xros \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
         -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
@@ -546,7 +534,6 @@ build_tvos_sim() {
         -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
         -DCMAKE_SYSTEM_NAME=tvOS \
         -DCMAKE_OSX_SYSROOT=appletvsimulator \
-        -DGGML_METAL_TARGET_OS=tvos \
         -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
         -DGGML_METAL=ON \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
@@ -565,7 +552,6 @@ build_tvos_device() {
         -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
         -DCMAKE_SYSTEM_NAME=tvOS \
         -DCMAKE_OSX_SYSROOT=appletvos \
-        -DGGML_METAL_TARGET_OS=tvos \
         -DCMAKE_OSX_ARCHITECTURES="arm64" \
         -DGGML_METAL=ON \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \

ci/README-MUSA.md

@@ -21,13 +21,13 @@ docker run --privileged -it \
     -v $HOME/llama.cpp/ci-cache:/ci-cache \
     -v $HOME/llama.cpp/ci-results:/ci-results \
     -v $PWD:/ws -w /ws \
-    registry.mthreads.com/mcconline/musa_sdk:5.2.0-devel-ubuntu22.04-s5000
+    mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
 ```
 
 Inside the container, execute the following commands:
 
 ```bash
-apt update -y && apt install -y bc cmake ccache git python3.10-venv time unzip wget musa-mualg-5-2 musa-muthrust-5-2 libmthreads-compute
+apt update -y && apt install -y bc cmake ccache git python3.10-venv time unzip wget
 git config --global --add safe.directory /ws
 GG_BUILD_MUSA=1 bash ./ci/run.sh /ci-results /ci-cache
 ```

ci/run.sh

@@ -49,7 +49,17 @@ mkdir -p "$2"
 OUT=$(realpath "$1")
 MNT=$(realpath "$2")
 
+# gpu-rocm self-hosted runner can't upload logs to blob; keep each run's logs in
+# their own dir keyed by the GitHub run id so an Actions run URL maps to its logs.
+if [ -n "${GG_BUILD_ROCM}" ] && [ -n "${GITHUB_RUN_ID}" ]; then
+    OUT="$OUT/run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
+    mkdir -p "$OUT"
+    echo "ci results dir: $OUT"
+fi
+
 rm -f $OUT/*.log
+rm -f $OUT/*.exit
+rm -f $OUT/*.md
 
 sd=`dirname $0`
 cd $sd/../
@@ -72,8 +82,8 @@ else
 fi
 
 if [ ! -z ${GG_BUILD_CUDA} ]; then
-    # TODO: Drop GGML_CUDA_CCCL_VERSION when CUDA CI uses CTK >= 13.5, which bundles CCCL >= 3.5.
-    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CCCL_VERSION=v3.4.3"
+    # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON -DGGML_CUDA_CUB_3DOT2=ON"
 
     if command -v nvidia-smi >/dev/null 2>&1; then
         CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
@@ -150,8 +160,8 @@ if [ ! -z ${GG_BUILD_WEBGPU} ]; then
 fi
 
 if [ ! -z ${GG_BUILD_MUSA} ]; then
-    # Use ph1 by default (MTT S5000)
-    MUSA_ARCH=${MUSA_ARCH:-31}
+    # Use qy1 by default (MTT S80)
+    MUSA_ARCH=${MUSA_ARCH:-21}
     CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
 fi
 
@@ -179,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
     fi
     CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
 
-    # TODO: fix failing tests on OpenVINO backend
-    CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
+    # TODO: fix and re-enable the `test-llama-archs` test below
+    CTEST_EXTRA="-E test-llama-archs|test-recurrent-state-rollback-nemotron-h"
 fi
 
 ## helpers
@@ -201,6 +211,10 @@ function gg_wget {
     cd $cwd
 }
 
+function gg_printf {
+    printf -- "$@" >> $OUT/README.md
+}
+
 function gg_run {
     ci=$1
 
@@ -209,10 +223,13 @@ function gg_run {
 
     gg_run_$ci | tee $OUT/$ci.log
     cur=$?
+    echo "$cur" > $OUT/$ci.exit
 
     set +x
     set +o pipefail
 
+    gg_sum_$ci
+
     ret=$((ret | cur))
 }
 
@@ -233,11 +250,22 @@ function gg_run_ctest_debug {
     (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
     (time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
 
-    (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
+    (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
 
     set +e
 }
 
+function gg_sum_ctest_debug {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Runs ctest in debug mode\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
+    gg_printf '```\n'
+    gg_printf '\n'
+}
+
 # ctest_release
 
 function gg_run_ctest_release {
@@ -262,6 +290,16 @@ function gg_run_ctest_release {
     set +e
 }
 
+function gg_sum_ctest_release {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Runs ctest in release mode\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
+    gg_printf '```\n'
+}
+
 # test_llama_archs_tensor_split
 
 function gg_run_test_llama_archs_tensor_split {
@@ -286,23 +324,14 @@ function gg_run_test_llama_archs_tensor_split {
     set +e
 }
 
-# test_llama_archs_models
+function gg_sum_test_llama_archs_tensor_split {
+    gg_printf '### %s\n\n' "${ci}"
 
-function gg_run_test_llama_archs_models {
-    cd ${SRC}
-
-    set -e
-
-    # TODO: fix and re-enable `test-llama-archs` on OpenVINO
-    # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
-    if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
-        rm -rf build-ci-models && mkdir -p build-ci-models
-
-        # generate the dummy models used by the model-dependent tests
-        ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
-    fi
-
-    set +e
+    gg_printf 'Runs test-llama-archs with 1 to 4 devices\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}.log)"
+    gg_printf '```\n'
 }
 
 # test_scripts
@@ -318,6 +347,17 @@ function gg_run_test_scripts {
     set +e
 }
 
+function gg_sum_test_scripts {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Runs test scripts\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}-scripts.log)"
+    gg_printf '```\n'
+    gg_printf '\n'
+}
+
 function gg_get_model {
     #local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-f16.gguf"
     local gguf_0="$MNT/models/qwen3/0.6B/ggml-model-q4_0.gguf"
@@ -361,6 +401,26 @@ function gg_run_ctest_with_model_release {
     cd ..
 }
 
+function gg_sum_ctest_with_model_debug {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Runs ctest with model files in debug mode\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
+    gg_printf '```\n'
+}
+
+function gg_sum_ctest_with_model_release {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Runs ctest with model files in release mode\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
+    gg_printf '```\n'
+}
+
 # qwen3_0_6b
 
 function gg_run_qwen3_0_6b {
@@ -465,24 +525,50 @@ function gg_run_qwen3_0_6b {
         return 0
     }
 
-    check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")"
+    check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
     if [ -z ${GG_BUILD_NO_BF16} ]; then
-        check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")"
+        check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
     fi
-    check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")"
-    check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")"
-    check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")"
-    check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")"
-    check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")"
-   #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" # note: ppl > 20.0 for this quant and model
-    check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")"
-    check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")"
-    check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")"
-    check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")"
+    check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+    check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+    check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+    check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+    check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+   #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log # note: ppl > 20.0 for this quant and model
+    check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+    check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+    check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+    check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
+
+    cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log
 
     set +e
 }
 
+function gg_sum_qwen3_0_6b {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Qwen3 0.6B:\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
+    gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)"
+    gg_printf '- f16:\n```\n%s\n```\n'  "$(cat $OUT/${ci}-tg-f16.log)"
+    if [ -z ${GG_BUILD_NO_BF16} ]; then
+        gg_printf '- bf16:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-bf16.log)"
+    fi
+    gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
+    gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
+    gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
+    gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
+    gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
+    gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
+    gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
+    gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
+    gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
+    gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
+    gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)"
+}
+
 # bge-small
 
 function gg_run_embd_bge_small {
@@ -524,6 +610,15 @@ function gg_run_embd_bge_small {
     set +e
 }
 
+function gg_sum_embd_bge_small {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'BGE Small (BERT):\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
+    gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
+}
+
 # rerank_tiny
 
 function gg_run_rerank_tiny {
@@ -580,102 +675,88 @@ function gg_run_rerank_tiny {
     set +e
 }
 
+function gg_sum_rerank_tiny {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Rerank Tiny (Jina):\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-rk-f16.log)"
+}
+
 function gg_check_build_requirements {
     if ! command -v git &> /dev/null; then
-        echo 'git not found, please install'
+        gg_printf 'git not found, please install\n'
         exit 1
     fi
 
     if ! command -v git-lfs &> /dev/null; then
-        echo 'git-lfs not found, please install'
+        gg_printf 'git-lfs not found, please install\n'
         exit 1
     fi
 
     if ! git config --get filter.lfs.clean &> /dev/null; then
-        echo 'git-lfs not initialized, please run `git lfs install`'
+        gg_printf 'git-lfs not initialized, please run `git lfs install`\n'
         exit 1
     fi
 
     if ! command -v wget &> /dev/null; then
-        echo 'wget not found, please install'
+        gg_printf 'wget not found, please install\n'
         exit 1
     fi
 
     if ! command -v python3 &> /dev/null; then
-        echo 'python3 not found, please install'
+        gg_printf 'python3 not found, please install\n'
         exit 1
     fi
 
     if ! command -v pip3 &> /dev/null; then
-        echo 'pip3 not found, please install'
+        gg_printf 'pip3 not found, please install\n'
         exit 1
     fi
 
     if ! python3 -m ensurepip --help &> /dev/null; then
-        echo 'ensurepip not found, please install python3-venv package'
+        gg_printf 'ensurepip not found, please install python3-venv package\n'
         exit 1
     fi
 
     if ! command -v cmake &> /dev/null; then
-        echo 'cmake not found, please install'
+        gg_printf 'cmake not found, please install\n'
         exit 1
     fi
 
     if ! command -v ccache &> /dev/null; then
-        echo 'ccache not found, please consider installing for faster builds'
+        gg_printf 'ccache not found, please consider installing for faster builds\n'
     fi
 
     if ! command -v ctest &> /dev/null; then
-        echo 'ctest not found, please install'
-        exit 1
-    fi
-
-    if ! command -v unzip &> /dev/null; then
-        echo 'unzip not found, please install'
+        gg_printf 'ctest not found, please install\n'
         exit 1
     fi
 }
 
-function gg_run_test_backend_ops {
+function gg_run_test_backend_ops_cpu {
     cd ${SRC}
 
     cd build-ci-release
 
     set -e
 
-    local n_jobs=$(nproc)
-    if [ "${n_jobs}" -gt 2 ]; then
-        n_jobs=2
-    fi
-    local args_extra="-j ${n_jobs}"
-
-    # TODO: fix multi-threaded for ROCm
-    #       https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
-    if [ ! -z ${GG_BUILD_ROCM} ]; then
-        args_extra=""
-    fi
-
-    # TODO: MoltenVK bug?
-    #       https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
-    if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
-        args_extra=""
-    fi
-
-    # TODO: OpenVINO GPU plugin crashes (CL_OUT_OF_RESOURCES) with 2 concurrent workers on GPU.
-    if [ ! -z "${GG_BUILD_OPENVINO}" ] && [ "${GGML_OPENVINO_DEVICE:-}" = "GPU" ]; then
-        args_extra=""
-    fi
-
-    # TODO: reduce the test-backend-ops timeout to 1800s
-    if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
-        (time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
-    else
-        (time timeout 3600 ./bin/test-backend-ops ${args_extra}       ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
-    fi
+    (time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log
 
     set +e
 }
 
+function gg_sum_test_backend_ops_cpu {
+    gg_printf '### %s\n\n' "${ci}"
+
+    gg_printf 'Runs test-backend-ops for CPU backend\n'
+    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
+    gg_printf '```\n'
+    gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)"
+    gg_printf '```\n'
+    gg_printf '\n'
+}
+
 ## main
 
 export LLAMA_ARG_LOG_PREFIX=1
@@ -704,11 +785,12 @@ ret=0
 test $ret -eq 0 && gg_run ctest_debug
 test $ret -eq 0 && gg_run ctest_release
 
-test $ret -eq 0 && gg_run test_backend_ops
-
-test $ret -eq 0 && gg_run test_llama_archs_models
 test $ret -eq 0 && gg_run test_llama_archs_tensor_split
 
+if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
+    test $ret -eq 0 && gg_run test_backend_ops_cpu
+fi
+
 if [ -z ${GG_BUILD_LOW_PERF} ]; then
     test $ret -eq 0 && gg_run embd_bge_small
     test $ret -eq 0 && gg_run rerank_tiny
@@ -723,4 +805,6 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then
     test $ret -eq 0 && gg_run ctest_with_model_release
 fi
 
+cat $OUT/README.md
+
 exit $ret

cmake/llama-config.cmake.in

@@ -17,16 +17,14 @@ find_library(llama_LIBRARY llama
     NO_CMAKE_FIND_ROOT_PATH
 )
 
-if(NOT TARGET llama)
-    add_library(llama UNKNOWN IMPORTED)
-    set_target_properties(llama
-        PROPERTIES
-            INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
-            INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;"
-            IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
-            IMPORTED_LOCATION "${llama_LIBRARY}"
-            INTERFACE_COMPILE_FEATURES c_std_90
-            POSITION_INDEPENDENT_CODE ON)
-endif()
+add_library(llama UNKNOWN IMPORTED)
+set_target_properties(llama
+    PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${LLAMA_INCLUDE_DIR}"
+        INTERFACE_LINK_LIBRARIES "ggml::ggml;ggml::ggml-base;"
+        IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+        IMPORTED_LOCATION "${llama_LIBRARY}"
+        INTERFACE_COMPILE_FEATURES c_std_90
+        POSITION_INDEPENDENT_CODE ON)
 
 check_required_components(Llama)

common/CMakeLists.txt

@@ -53,10 +53,7 @@ endif()
 
 set(TARGET llama-common)
 
-include(parsers/sources.cmake)
-
 add_library(${TARGET}
-    ${LLAMA_CHAT_PARSERS_SOURCES}
     arg.cpp
     arg.h
     base64.hpp
@@ -84,8 +81,6 @@ add_library(${TARGET}
     imatrix-loader.cpp
     imatrix-loader.h
     json-schema-to-grammar.cpp
-    json-schema.cpp
-    json-schema.h
     json.cpp
     json.h
     llguidance.cpp

common/arg.cpp

@@ -351,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
 static std::string get_default_local_path(const std::string & url) {
     auto f = string_split<std::string>(url, '#').front();
     f = string_split<std::string>(f, '?').front();
-    return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
+    return fs_get_cache_file(string_split<std::string>(f, '/').back());
 }
 
 static bool spec_types_is_default(const common_params & params) {
@@ -387,9 +387,6 @@ common_models_handler common_models_handler_init(const common_params & params, l
             break;
         }
     }
-    if (curr_ex == LLAMA_EXAMPLE_DOWNLOAD) {
-        use_mmproj = true;
-    }
 
     opts.bearer_token    = params.hf_token;
     opts.offline         = params.offline;
@@ -720,24 +717,24 @@ void common_models_handler_apply(common_models_handler & handler, common_params
 // 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows)
 // 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows)
 static void common_params_apply_system_config(common_params & params, llama_example ex) {
-    std::vector<std::filesystem::path> paths;
+    std::vector<std::string> paths;
 
 #if defined(_WIN32)
-    const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA");
+    const std::string program_data = common_get_env("PROGRAMDATA");
     if (!program_data.empty()) {
-        paths.push_back(program_data / "llama.cpp" / "config.ini");
+        paths.push_back(program_data + "\\llama.cpp\\config.ini");
     }
 #else
     paths.push_back("/etc/llama.cpp/config.ini");
 #endif
 
     try {
-        paths.push_back(fs_get_config_directory() / "config.ini");
+        paths.push_back(fs_get_config_directory() + "config.ini");
     } catch (const std::exception & e) {
         LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what());
     }
 
-    std::vector<std::filesystem::path> found;
+    std::vector<std::string> found;
     for (const auto & path : paths) {
         std::error_code ec;
         if (std::filesystem::exists(path, ec)) {
@@ -751,7 +748,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
     common_preset_context ctx(ex);
     ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
     for (const auto & path : found) {
-        LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
+        LOG_INF("using config file: %s\n", path.c_str());
         common_preset global;
         common_presets presets = ctx.load_from_ini(path, global);
         global.apply_to_params(params);
@@ -875,6 +872,17 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
                     arg.c_str(), e.what(), opt.to_string().c_str()));
             }
         }
+
+        // TODO: remove this check after deprecating --mmap|mlock|dio
+        auto has_arg = [&](std::initializer_list<const char *> names) {
+            return std::any_of(names.begin(), names.end(), [&](const char * name) {
+                return seen_args.count(name);
+            });
+        };
+        if (has_arg({"-lm", "--load-mode"}) &&
+            has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) {
+            LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n");
+        }
     };
 
     // parse all CLI args now, so that -hf is available below for remote preset resolution
@@ -886,12 +894,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
     postprocess_cpu_params(params.speculative.draft.cpuparams,       &params.cpuparams);
     postprocess_cpu_params(params.speculative.draft.cpuparams_batch, &params.cpuparams_batch);
 
-    // default the mmproj device to the global device selection if not set explicitly with -mmdev
-    if (params.mmproj_use_gpu && params.mmproj_device == nullptr && !params.devices.empty()) {
-        params.mmproj_device = params.devices.front();
-        params.mmproj_use_gpu = params.mmproj_device != nullptr;
-    }
-
     if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {
         throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
     }
@@ -958,11 +960,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
         ));
     }
 
-    // if the preserve_reasoning kwarg was not specified explicitly, enable it by default
-    if (!params.default_template_kwargs.count("preserve_reasoning")) {
-        params.default_template_kwargs["preserve_reasoning"] = "true";
-    }
-
     return true;
 }
 
@@ -2019,7 +2016,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.sampling.temp = std::max(params.sampling.temp, 0.0f);
             params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP;
         }
-    ).set_sampling().set_env("LLAMA_ARG_TEMPERATURE"));
+    ).set_sampling());
     add_opt(common_arg(
         {"--top-k"}, "N",
         string_format("top-k sampling (default: %d, 0 = disabled)", params.sampling.top_k),
@@ -2035,7 +2032,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.sampling.top_p = std::stof(value);
             params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TOP_P;
         }
-    ).set_sampling().set_env("LLAMA_ARG_TOP_P"));
+    ).set_sampling());
     add_opt(common_arg(
         {"--min-p"}, "N",
         string_format("min-p sampling (default: %.2f, 0.0 = disabled)", (double)params.sampling.min_p),
@@ -2043,7 +2040,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.sampling.min_p = std::stof(value);
             params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_MIN_P;
         }
-    ).set_sampling().set_env("LLAMA_ARG_MIN_P"));
+    ).set_sampling());
     add_opt(common_arg(
         {"--top-nsigma", "--top-n-sigma"}, "N",
         string_format("top-n-sigma sampling (default: %.2f, -1.0 = disabled)", params.sampling.top_n_sigma),
@@ -2099,7 +2096,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.sampling.penalty_repeat = penalty_repeat;
             params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT;
         }
-    ).set_sampling().set_env("LLAMA_ARG_REPEAT_PENALTY"));
+    ).set_sampling());
     add_opt(common_arg(
         {"--presence-penalty"}, "N",
         string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present),
@@ -2110,7 +2107,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
             params.sampling.penalty_present = penalty_present;
         }
-    ).set_sampling().set_env("LLAMA_ARG_PRESENCE_PENALTY"));
+    ).set_sampling());
     add_opt(common_arg(
         {"--frequency-penalty"}, "N",
         string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq),
@@ -2121,7 +2118,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
             params.sampling.penalty_freq = penalty_freq;
         }
-    ).set_sampling().set_env("LLAMA_ARG_FREQUENCY_PENALTY"));
+    ).set_sampling());
     add_opt(common_arg(
         {"--dry-multiplier"}, "N",
         string_format("set DRY sampling multiplier (default: %.2f, 0.0 = disabled)", (double)params.sampling.dry_multiplier),
@@ -2280,14 +2277,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_sampling());
     add_opt(common_arg(
         {"-j", "--json-schema"}, "SCHEMA",
-        "JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
+        "JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
         [](common_params & params, const std::string & value) {
             params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, json_schema_to_grammar(json::parse(value))};
         }
     ).set_sampling());
     add_opt(common_arg(
         {"-jf", "--json-schema-file"}, "FILE",
-        "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
+        "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
         [](common_params & params, const std::string & value) {
             std::ifstream file(value);
             if (!file) {
@@ -2608,7 +2605,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     add_opt(common_arg(
         // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
         {"-mmdev", "--mmproj-device"}, "DEVICE",
-        "device to use for multimodal projector (none = don't offload, default: follows --device)\n"
+        "device to use for multimodal projector (none = don't offload, default: auto)\n"
         "use --list-devices to see a list of available devices",
         [](common_params & params, const std::string & value) {
             if (value == "none") {
@@ -2676,17 +2673,42 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.video_ffmpeg_bin_dir = value;
         }
     ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));
-    add_opt(common_arg(
-        {"--rpc"}, "SERVERS",
-        "comma-separated list of RPC servers (host:port)",
-        [](common_params & params, const std::string & value) {
-            if (!llama_supports_rpc()) {
-                throw std::invalid_argument("RPC not supported in this build");
+    if (params.is_gen_docs || llama_supports_rpc()) {
+        add_opt(common_arg(
+            {"--rpc"}, "SERVERS",
+            "comma-separated list of RPC servers (host:port)",
+            [](common_params & params, const std::string & value) {
+                add_rpc_devices(value);
+                GGML_UNUSED(params);
             }
-            add_rpc_devices(value);
-            GGML_UNUSED(params);
+        ).set_env("LLAMA_ARG_RPC"));
+    }
+    add_opt(common_arg(
+        {"--mlock"},
+        "DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing",
+        [](common_params & params) {
+            LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");
+            params.load_mode = LLAMA_LOAD_MODE_MLOCK;
         }
-    ).set_env("LLAMA_ARG_RPC"));
+    ).set_env("LLAMA_ARG_MLOCK"));
+    add_opt(common_arg(
+        {"--mmap"},
+        {"--no-mmap"},
+        "DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)",
+        [](common_params & params, bool value) {
+            LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n");
+            params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE;
+        }
+    ).set_env("LLAMA_ARG_MMAP"));
+    add_opt(common_arg(
+        {"-dio", "--direct-io"},
+        {"-ndio", "--no-direct-io"},
+        "DEPRECATED in favor of `--load-mode`: use DirectIO if available",
+        [](common_params & params, bool value) {
+            LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n");
+            params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE;
+        }
+    ).set_env("LLAMA_ARG_DIO"));
     add_opt(common_arg(
         {"-lm", "--load-mode"}, "MODE",
         "model loading mode (default: auto)\n"
@@ -2707,18 +2729,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         }
     ).set_env("LLAMA_ARG_LOAD_MODE"));
     add_opt(common_arg(
-        {"-lzm", "--lazy-mode"}, "MODE",
+        {"--tensor-read-lazy"}, "MODE",
         "on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
         "- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
         "- auto: on, but only for tensors larger than 4 GiB\n"
         "- off: always keep them resident",
         [](common_params & params, const std::string & value) {
-            /**/ if (value == "on")   { params.lazy_mode = LLAMA_LAZY_MODE_ON;   }
-            else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
-            else if (value == "off")  { params.lazy_mode = LLAMA_LAZY_MODE_OFF;  }
+            /**/ if (value == "on")   { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON;   }
+            else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
+            else if (value == "off")  { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;  }
             else { throw std::invalid_argument("invalid value"); }
         }
-    ).set_env("LLAMA_ARG_LAZY_MODE"));
+    ).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));
     add_opt(common_arg(
         {"--numa"}, "TYPE",
         "attempt optimizations that help on some NUMA systems\n"
@@ -3312,18 +3334,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_examples({LLAMA_EXAMPLE_EMBEDDING}));
     add_opt(common_arg(
         {"--host"}, "HOST",
-        string_format("IP addresses to listen on, comma-separated, or UNIX socket paths ending in .sock; with multiple TCP addresses, :: binds IPv6 only; overlapping addresses result in undefined behavior (default: %s)", params.hostnames[0].c_str()),
+        string_format("ip address to listen, or bind to an UNIX socket if the address ends with .sock (default: %s)", params.hostname.c_str()),
         [](common_params & params, const std::string & value) {
-            params.hostnames.clear();
-            for (auto & host : parse_csv_row(value)) {
-                host = string_strip(host);
-                if (!host.empty()) {
-                    params.hostnames.push_back(host);
-                }
-            }
-            if (params.hostnames.empty()) {
-                throw std::invalid_argument("--host requires at least one address");
-            }
+            params.hostname = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_HOST"));
     add_opt(common_arg(
@@ -3540,10 +3553,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
                     LOG_WRN("Setting 'enable_thinking' via --chat-template-kwargs is deprecated. "
                             "Use --reasoning on / --reasoning off instead.\n");
                 }
-                if (item.key() == "preserve_reasoning") {
-                    LOG_WRN("Setting 'preserve_reasoning' via --chat-template-kwargs is deprecated. "
-                            "Use --reasoning-preserve / --no-reasoning-preserve instead.\n");
-                }
                 params.default_template_kwargs[item.key()] = item.value().dump();
             }
         }
@@ -3734,7 +3743,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     add_opt(common_arg(
         {"--reasoning-preserve"},
         {"--no-reasoning-preserve"},
-        "preserve reasoning trace in the full history, not just the last assistant message (default: enabled)\n"
+        "preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n"
         "compatible with certain templates having 'supports_preserve_reasoning' capability\n"
         "example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking",
         [](common_params & params, bool value) {
@@ -3743,7 +3752,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             } else {
                 params.default_template_kwargs["preserve_reasoning"] = "false";
             }
-            params.preserve_reasoning_specified = true;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE"));
     add_opt(common_arg(
@@ -3883,14 +3891,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             common_log_set_file(common_log_main(), value.c_str());
         }
     ).set_env("LLAMA_ARG_LOG_FILE"));
-    add_opt(common_arg(
-        {"--log-jsonl"},
-        {"--no-log-jsonl"},
-        "Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)",
-        [](common_params &, bool value) {
-            common_log_set_jsonl(value);
-        }
-    ).set_env("LLAMA_ARG_LOG_JSONL"));
     add_opt(common_arg(
         {"--log-prompts-dir"}, "PATH",
         "Log prompts to directory (auto-created if not present; only used for debugging, default: disabled)",
@@ -4211,7 +4211,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING"));
     add_opt(common_arg(
         {"--spec-draft-device", "-devd", "--device-draft"}, "<dev1,dev2,..>",
-        "comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)\n"
+        "comma-separated list of devices to use for offloading the draft model (none = don't offload)\n"
         "use --list-devices to see a list of available devices",
         [](common_params & params, const std::string & value) {
             params.speculative.draft.devices = parse_device_list(value);

common/arg.h

@@ -122,8 +122,6 @@ struct common_params_context {
 
 // parse input arguments from CLI
 // if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message)
-// TODO: this function can load ggml backend (by calling llama_support_rpc)
-//       this is a side-effect that should be avoided
 bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr);
 
 // load all backends and print the list of available (non-CPU) devices to stdout

common/build-info.cpp.in

@@ -29,7 +29,7 @@ const char * llama_build_info(void) {
     return s.c_str();
 }
 
-void llama_print_build_info(const char * llama_version, FILE * stream) {
-    fprintf(stream, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
-    fprintf(stream, "built with %s for %s\n", llama_compiler(), llama_build_target());
+void llama_print_build_info(const char * llama_version) {
+    fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
+    fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
 }

common/build-info.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <cstdio>
-
 int llama_build_number(void);
 
 const char * llama_commit(void);
@@ -10,4 +8,4 @@ const char * llama_compiler(void);
 const char * llama_build_target(void);
 const char * llama_build_info(void);
 
-void llama_print_build_info(const char *, FILE * = stderr);
+void llama_print_build_info(const char *);

common/chat-auto-parser-generator.cpp

@@ -5,7 +5,6 @@
 #include "common.h"
 #include "json-schema-to-grammar.h"
 #include "log.h"
-#include "parsers/parsers.h"
 #include "peg-parser.h"
 
 #include <stdexcept>
@@ -13,6 +12,16 @@
 
 using json = common_json;
 
+// Helper to iterate over tools/functions
+static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
+    for (const auto & tool : tools) {
+        if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
+            continue;
+        }
+        fn(tool);
+    }
+}
+
 namespace autoparser {
 
 parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :
@@ -78,6 +87,15 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
     if (include_grammar) {
         data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
         data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
+            foreach_function(inputs.tools, [&](const json & tool) {
+                const auto & function = tool.at("function");
+                auto         schema   = function.contains("parameters") ? function.at("parameters") : json::object();
+                builder.resolve_refs(schema);
+            });
+            if (has_response_format) {
+                auto schema = inputs.json_schema;
+                builder.resolve_refs(schema);
+            }
             parser.build_grammar(builder, data.grammar_lazy);
         });
 
@@ -294,7 +312,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context
     foreach_function(inputs.tools, [&](const json & tool) {
         const auto & func   = tool.at("function");
         std::string  name   = func.at("name");
-        const auto   schema = common_chat_tool_parameters(func);
+        const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object();
 
         // Build call_id parser based on position (if supported)
         bool have_call_id = false;
@@ -365,31 +383,43 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
     common_peg_parser tool_choice = p.choice();
 
     foreach_function(inputs.tools, [&](const json & tool) {
-        const auto & func = tool.at("function");
-        std::string  name = func.at("name");
+        const auto &          func       = tool.at("function");
+        std::string           name       = func.at("name");
+        auto                  params     = func.contains("parameters") ? func.at("parameters") : json::object();
+        const auto &          properties = params.contains("properties") ? params.at("properties") : json::object();
+
+        std::set<std::string> required;
+        if (params.contains("required")) {
+            required = params.at("required").get<std::set<std::string>>();
+        }
+
+        auto schema_info = common_schema_info();
+        schema_info.resolve_refs(params);
 
         // Build parser for each argument, separating required and optional
         std::vector<common_peg_parser> required_parsers;
         std::vector<common_peg_parser> optional_parsers;
-        foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
+        for (const auto & [param_name, param_schema] : properties.items()) {
+            bool is_required = required.find(param_name) != required.end();
+
             auto arg =
-                p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
+                p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) +
                                            arguments.name_suffix) +
                            arguments.value_prefix +
-                           (param.schema->may_be_string() ?
+                           (schema_info.resolves_to_string(param_schema) ?
                                 p.ac(p.tool_arg_string_value(until_suffix) +
                                     p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
                                 (p.tool_arg_json_value(p.schema(
-                                    p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) +
+                                    p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +
                                     p.tool_arg_close(p.literal(arguments.value_suffix)))));
 
-            auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
-            if (param.required) {
+            auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
+            if (is_required) {
                 required_parsers.push_back(named_arg);
             } else {
                 optional_parsers.push_back(named_arg);
             }
-        });
+        }
 
         // Build required arg sequence in definition order
         common_peg_parser args_seq = p.eps();

common/chat-peg-parser.cpp

@@ -318,13 +318,13 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
     bool is_content   = node.tag == common_chat_peg_builder::CONTENT;
 
     if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here
-        result.reasoning_content += node.sanitized_text();
+        result.reasoning_content += std::string(node.text);
     }
 
     if (is_content) {
         // Concatenate content from multiple content nodes (e.g., when reasoning markers
         // are preserved before content markers in reasoning_format=NONE mode)
-        result.content += node.sanitized_text();
+        result.content += std::string(node.text);
     }
 
     // Handle tool-related tags (supporting both JSON and tagged formats)
@@ -488,7 +488,7 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         // Build argument parsers
         auto args = eps();
@@ -565,7 +565,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         auto args = eps();
         if (params.contains("properties") && !params["properties"].empty()) {
@@ -640,7 +640,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         // Build inner object fields
         std::vector<common_peg_parser> inner_fields;
@@ -726,7 +726,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +
                           atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
@@ -795,7 +795,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         auto tool_name_ = name_key_parser + space() + literal(":") + space() +
                          atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
@@ -1058,12 +1058,12 @@ void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, co
     const auto & node = arena.get(id);
 
     if (node.tag == "reasoning") {
-        result.reasoning_content += node.sanitized_text();
+        result.reasoning_content += std::string(node.text);
         return;
     }
 
     if (node.tag == "content") {
-        result.content += node.sanitized_text();
+        result.content += std::string(node.text);
         return;
     }
 
@@ -1206,12 +1206,12 @@ void common_chat_peg_minimax_m3_mapper::visit(const common_peg_ast_arena & arena
     const auto & node = arena.get(id);
 
     if (node.tag == common_chat_peg_builder::REASONING) {
-        result.reasoning_content += node.sanitized_text();
+        result.reasoning_content += std::string(node.text);
         return;
     }
 
     if (node.tag == common_chat_peg_builder::CONTENT) {
-        result.content += node.sanitized_text();
+        result.content += std::string(node.text);
         return;
     }
 

common/chat.h

@@ -360,9 +360,6 @@ common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg
 
 common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
 
-// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments.
-common_json common_chat_tool_parameters(const common_json & function);
-
 // get template caps, useful for reporting to server /props endpoint
 std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
 

common/common.cpp

@@ -46,10 +46,11 @@
 #include <io.h>
 #else
 #include <sys/ioctl.h>
+#include <sys/stat.h>
 #include <unistd.h>
 #endif
 
-#if !defined(_WIN32)
+#if defined(__linux__)
 #include <sys/types.h>
 #include <pwd.h>
 #endif
@@ -613,6 +614,34 @@ std::string string_from(const struct llama_context * ctx, const std::vector<llam
     return buf.str();
 }
 
+std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch) {
+    std::stringstream buf;
+
+    buf << "[ ";
+
+    bool first = true;
+    for (int i = 0; i < batch.n_tokens; ++i) {
+        if (!first) {
+            buf << ", ";
+        } else {
+            first = false;
+        }
+
+        auto detokenized = common_token_to_piece(ctx, batch.token[i]);
+
+        buf << "\n"          << std::to_string(i)
+            << ", token '"   << detokenized << "'"
+            << ", pos "      << std::to_string(batch.pos[i])
+            << ", n_seq_id " << std::to_string(batch.n_seq_id[i])
+            << ", seq_id "   << std::to_string(batch.seq_id[i][0])
+            << ", logits "   << std::to_string(batch.logits[i]);
+    }
+
+    buf << " ]";
+
+    return buf.str();
+}
+
 void string_process_escapes(std::string & input) {
     std::size_t input_len = input.length();
     std::size_t output_idx = 0;
@@ -871,7 +900,7 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
 
 
 #ifdef _WIN32
-std::wstring utf8_to_wstring(const std::string & str) {
+static std::wstring utf8_to_wstring(const std::string & str) {
     if (str.empty()) {
         return std::wstring();
     }
@@ -887,52 +916,82 @@ std::wstring utf8_to_wstring(const std::string & str) {
 
     return wstr;
 }
-
-std::string wstring_to_utf8(const std::wstring & str) {
-    if (str.empty()) {
-        return std::string();
-    }
-
-    int size = WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0, NULL, NULL);
-
-    if (size <= 0) {
-        return std::string();
-    }
-
-    std::string utf8(size, 0);
-    WideCharToMultiByte(CP_UTF8, 0, str.c_str(), (int)str.size(), &utf8[0], size, NULL, NULL);
-
-    return utf8;
-}
 #endif
 
-// returns the path as a UTF-8 string, preserving its separators
-std::string fs_path_to_utf8(const std::filesystem::path & path) {
-    const auto value = path.u8string();
-    return std::string(value.begin(), value.end());
-}
+// returns true if successful, false otherwise
+bool fs_create_directory_with_parents(const std::string & path) {
+#ifdef _WIN32
+    std::wstring wpath = utf8_to_wstring(path);
 
-void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
-    std::error_code ec;
-    std::filesystem::path path_tmp = path;
-    path_tmp += ".tmp";
-
-    if (path.has_parent_path()) {
-        std::filesystem::create_directories(path.parent_path(), ec);
+    // if the path already exists, check whether it's a directory
+    const DWORD attributes = GetFileAttributesW(wpath.c_str());
+    if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+        return true;
     }
 
-    std::ofstream file(path_tmp, std::ios::binary);
-    file << data;
-    file.close();
+    size_t pos_slash = 0;
 
-    if (!file.fail()) {
-        std::filesystem::rename(path_tmp, path, ec);
+    // process path from front to back, procedurally creating directories
+    while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {
+        const std::wstring subpath = wpath.substr(0, pos_slash);
+
+        pos_slash += 1;
+
+        // skip the drive letter, in some systems it can return an access denied error
+        if (subpath.length() == 2 && subpath[1] == ':') {
+            continue;
+        }
+
+        const bool success = CreateDirectoryW(subpath.c_str(), NULL);
+
+        if (!success) {
+            const DWORD error = GetLastError();
+
+            // if the path already exists, ensure that it's a directory
+            if (error == ERROR_ALREADY_EXISTS) {
+                const DWORD attributes = GetFileAttributesW(subpath.c_str());
+                if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {
+                    return false;
+                }
+            } else {
+                return false;
+            }
+        }
     }
 
-    if (file.fail() || ec) {
-        std::filesystem::remove(path_tmp, ec);
-        throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
+    return true;
+#else
+    // if the path already exists, check whether it's a directory
+    struct stat info;
+    if (stat(path.c_str(), &info) == 0) {
+        return S_ISDIR(info.st_mode);
     }
+
+    size_t pos_slash = 1; // skip leading slashes for directory creation
+
+    // process path from front to back, procedurally creating directories
+    while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {
+        const std::string subpath = path.substr(0, pos_slash);
+        struct stat info;
+
+        // if the path already exists, ensure that it's a directory
+        if (stat(subpath.c_str(), &info) == 0) {
+            if (!S_ISDIR(info.st_mode)) {
+                return false;
+            }
+        } else {
+            // create parent directories
+            const int ret = mkdir(subpath.c_str(), 0755);
+            if (ret != 0) {
+                return false;
+            }
+        }
+
+        pos_slash += 1;
+    }
+
+    return true;
+#endif // _WIN32
 }
 
 bool fs_is_directory(const std::string & path) {
@@ -957,77 +1016,113 @@ void common_set_env(const std::string & name, const std::string & value) {
 #endif
 }
 
-std::filesystem::path common_get_path_from_env(const std::string & name) {
-#if defined(_WIN32)
-    const std::wstring wname = utf8_to_wstring(name);
-    const wchar_t * wvalue = _wgetenv(wname.c_str());
-    return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
-#else
-    const char * value = std::getenv(name.c_str());
-    return value ? std::filesystem::path(value) : std::filesystem::path();
-#endif
-}
-
-#if !defined(_WIN32)
-static std::filesystem::path get_home_directory() {
-    std::filesystem::path home = common_get_path_from_env("HOME");
-    if (!home.empty()) {
-        return home;
-    }
-    const struct passwd * pw = getpwuid(getuid());
-    if (!pw || !pw->pw_dir || !*pw->pw_dir) {
-        throw std::runtime_error("Failed to find $HOME directory");
-    }
-    return pw->pw_dir;
-}
-#endif
-
-std::filesystem::path fs_get_cache_directory() {
-    std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");
-    if (!cache_directory.empty()) {
-        return cache_directory;
-    }
-#if defined(_WIN32)
-    cache_directory = common_get_path_from_env("LOCALAPPDATA");
+std::string fs_get_cache_directory() {
+    std::string cache_directory = "";
+    auto ensure_trailing_slash = [](std::string p) {
+        // Make sure to add trailing slash
+        if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
+            p += DIRECTORY_SEPARATOR;
+        }
+        return p;
+    };
+    cache_directory = common_get_env("LLAMA_CACHE");
     if (cache_directory.empty()) {
-        throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
-    }
+#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
+        defined(__OpenBSD__) || defined(__NetBSD__)
+        const std::string xdg_cache_home = common_get_env("XDG_CACHE_HOME");
+        const std::string home           = common_get_env("HOME");
+        if (!xdg_cache_home.empty()) {
+            cache_directory = xdg_cache_home;
+        } else if (!home.empty()) {
+            cache_directory = home + "/.cache/";
+        } else {
+#if defined(__linux__)
+            /* no $HOME is defined, fallback to getpwuid */
+            struct passwd *pw = getpwuid(getuid());
+            if ((!pw) || (!pw->pw_dir)) {
+                throw std::runtime_error("Failed to find $HOME directory");
+            }
+
+            cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");
+#else /* defined(__linux__) */
+            throw std::runtime_error("Failed to find $HOME directory");
+#endif /* defined(__linux__) */
+        }
 #elif defined(__APPLE__)
-    cache_directory = get_home_directory() / "Library/Caches";
+        cache_directory = common_get_env("HOME");
+        if (cache_directory.empty()) {
+            throw std::runtime_error("Failed to find $HOME directory");
+        }
+        cache_directory += "/Library/Caches/";
+#elif defined(_WIN32)
+        cache_directory = common_get_env("LOCALAPPDATA");
+        if (cache_directory.empty()) {
+            throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
+        }
+#elif defined(__EMSCRIPTEN__)
+        GGML_ABORT("not implemented on this platform");
 #else
-    cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
-    if (cache_directory.empty()) {
-        cache_directory = get_home_directory() / ".cache";
-    }
+#  error Unknown architecture
 #endif
-    return cache_directory / "llama.cpp";
+        cache_directory = ensure_trailing_slash(cache_directory);
+        cache_directory += "llama.cpp";
+    }
+    return ensure_trailing_slash(cache_directory);
 }
 
-std::filesystem::path fs_get_config_directory() {
-    std::filesystem::path config_directory;
-#if defined(_WIN32)
-    config_directory = common_get_path_from_env("APPDATA");
+std::string fs_get_config_directory() {
+    std::string config_directory = "";
+    auto ensure_trailing_slash = [](std::string p) {
+        if (p.empty() || p.back() != DIRECTORY_SEPARATOR) {
+            p += DIRECTORY_SEPARATOR;
+        }
+        return p;
+    };
+#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \
+        defined(__OpenBSD__) || defined(__NetBSD__) || defined(__APPLE__)
+    const std::string xdg_config_home = common_get_env("XDG_CONFIG_HOME");
+    const std::string home            = common_get_env("HOME");
+    if (!xdg_config_home.empty()) {
+        config_directory = xdg_config_home;
+    } else if (!home.empty()) {
+        config_directory = home + "/.config/";
+    } else {
+#if defined(__linux__)
+        /* no $HOME is defined, fallback to getpwuid */
+        struct passwd *pw = getpwuid(getuid());
+        if ((!pw) || (!pw->pw_dir)) {
+            throw std::runtime_error("Failed to find $HOME directory");
+        }
+
+        config_directory = std::string(pw->pw_dir) + std::string("/.config/");
+#else
+        throw std::runtime_error("Failed to find $HOME directory");
+#endif
+    }
+#elif defined(_WIN32)
+    config_directory = common_get_env("APPDATA");
     if (config_directory.empty()) {
         throw std::runtime_error("Failed to find %APPDATA% directory");
     }
+#elif defined(__EMSCRIPTEN__)
+    // caller decides what to do when there is no config directory
+    throw std::runtime_error("not implemented on this platform");
 #else
-    config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
-    if (config_directory.empty()) {
-        config_directory = get_home_directory() / ".config";
-    }
+#  error Unknown architecture
 #endif
-    return config_directory / "llama.cpp";
+    config_directory = ensure_trailing_slash(config_directory);
+    config_directory += "llama.cpp";
+    return ensure_trailing_slash(config_directory);
 }
 
-std::filesystem::path fs_get_cache_file(const std::string & filename) {
+std::string fs_get_cache_file(const std::string & filename) {
     GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
-    const std::filesystem::path cache_directory = fs_get_cache_directory();
-    std::error_code ec;
-    std::filesystem::create_directories(cache_directory, ec);
-    if (ec) {
-        throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
+    std::string cache_directory = fs_get_cache_directory();
+    const bool success = fs_create_directory_with_parents(cache_directory);
+    if (!success) {
+        throw std::runtime_error("failed to create cache directory: " + cache_directory);
     }
-    return cache_directory / std::filesystem::u8path(filename);
+    return cache_directory + filename;
 }
 
 std::vector<common_file_info> fs_list(const std::string & path, bool include_directories) {
@@ -1432,8 +1527,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
         }
 
         if (llama_model_has_encoder(model)) {
-            common_batch batch = common_batch_get_one(lctx, tmp);
-            llama_process(lctx, LLAMA_PROCESS_TYPE_ENCODE, batch.get());
+            llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size()));
             llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
             if (decoder_start_token_id == LLAMA_TOKEN_NULL) {
                 decoder_start_token_id = bos;
@@ -1442,9 +1536,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
             tmp.push_back(decoder_start_token_id);
         }
         if (llama_model_has_decoder(model)) {
-            tmp.resize(std::min(tmp.size(), (size_t) params.n_batch));
-            common_batch batch = common_batch_get_one(lctx, tmp);
-            llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
+            llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch)));
         }
         llama_memory_clear(llama_get_memory(lctx), true);
         llama_synchronize(lctx);
@@ -1494,11 +1586,6 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
         return COMMON_CONTEXT_SEQ_RM_TYPE_NO;
     }
 
-    if (llama_n_rs_seq(ctx) > 0) {
-        COM_TRC("%s", "the context supports bounded partial sequence removal\n");
-        return COMMON_CONTEXT_SEQ_RM_TYPE_RS;
-    }
-
     common_context_seq_rm_type res = COMMON_CONTEXT_SEQ_RM_TYPE_PART;
 
     llama_memory_clear(mem, true);
@@ -1508,17 +1595,19 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
     tmp.push_back(0);
     tmp.push_back(0);
 
-    int ret;
-    {
-        common_batch batch = common_batch_get_one(ctx, tmp);
-        ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
-    }
+    int ret = llama_decode(ctx, llama_batch_get_one(tmp.data(), tmp.size()));
     if (ret != 0) {
-        COM_ERR("llama_process() failed: %d\n", ret);
+        COM_ERR("llama_decode() failed: %d\n", ret);
         res = COMMON_CONTEXT_SEQ_RM_TYPE_NO;
         goto done;
     }
 
+    if (llama_n_rs_seq(ctx) > 0) {
+        COM_TRC("%s", "the context supports bounded partial sequence removal\n");
+        res = COMMON_CONTEXT_SEQ_RM_TYPE_RS;
+        goto done;
+    }
+
     // try to remove the last tokens
     if (!llama_memory_seq_rm(mem, 0, 1, -1)) {
         COM_TRC("%s", "the context does not support partial sequence removal\n");
@@ -1599,7 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
     mparams.main_gpu        = params.main_gpu;
     mparams.split_mode      = params.split_mode;
     mparams.load_mode       = params.load_mode;
-    mparams.lazy_mode = params.lazy_mode;
+    mparams.tensor_read_lazy = params.tensor_read_lazy;
     mparams.tensor_split    = params.tensor_split;
     mparams.check_tensors   = params.check_tensors;
     mparams.use_extra_bufts = !params.no_extra_bufts;
@@ -1738,6 +1827,33 @@ void common_threadpools::init(llama_context * ctx, const common_params & params)
     llama_attach_threadpool(ctx, threadpool, threadpool_batch);
 }
 
+//
+// Batch utils
+//
+
+void common_batch_clear(struct llama_batch & batch) {
+    batch.n_tokens = 0;
+}
+
+void common_batch_add(
+                 struct llama_batch & batch,
+                        llama_token   id,
+                          llama_pos   pos,
+    const std::vector<llama_seq_id> & seq_ids,
+                               bool   logits) {
+    GGML_ASSERT(batch.seq_id[batch.n_tokens] && "llama_batch size exceeded");
+
+    batch.token   [batch.n_tokens] = id;
+    batch.pos     [batch.n_tokens] = pos;
+    batch.n_seq_id[batch.n_tokens] = seq_ids.size();
+    for (size_t i = 0; i < seq_ids.size(); ++i) {
+        batch.seq_id[batch.n_tokens][i] = seq_ids[i];
+    }
+    batch.logits  [batch.n_tokens] = logits;
+
+    batch.n_tokens++;
+}
+
 //
 // Vocab utils
 //
@@ -2074,137 +2190,18 @@ float lr_opt::get_lr(float epoch) const {
 }
 
 bool common_replay_last_token(struct llama_context * ctx, llama_token last_token, int32_t pos) {
-    common_batch batch(ctx);
-    batch.add(last_token, pos, 0, true);
-
-    if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
+    llama_batch batch = llama_batch_get_one(&last_token, 1);
+    batch.pos = &pos;
+    if (llama_decode(ctx, batch)) {
         LOG_ERR("%s: failed to replay last token\n", __func__);
         return false;
     }
     return true;
 }
 
-common_batch::common_batch(llama_context * ctx) : batch(llama_batch_ext_init(ctx)) {
-    const auto rope_type = llama_model_rope_type(llama_get_model(ctx));
-    n_pos = rope_type == LLAMA_ROPE_TYPE_MROPE || rope_type == LLAMA_ROPE_TYPE_IMROPE ? GGML_MROPE_SECTIONS : 1;
-}
-
-void common_batch::clear() {
-    tokens.clear();
-}
-
-int32_t common_batch::add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output) {
-    tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, {} });
-    return size() - 1;
-}
-
-int32_t common_batch::add(llama_token id, llama_pos pos, const std::vector<llama_seq_id> & seq_ids, bool output) {
-    GGML_ASSERT(!seq_ids.empty());
-
-    const int32_t idx = add(id, pos, seq_ids[0], output);
-    for (size_t s = 1; s < seq_ids.size(); ++s) {
-        add_seq(idx, seq_ids[s]);
-    }
-    return idx;
-}
-
-bool common_batch::add_seq(int32_t idx, llama_seq_id seq_id) {
-    if (idx < 0 || idx >= size()) {
-        return false;
-    }
-    tokens[idx].seq_ids_extra.push_back(seq_id);
-    return true;
-}
-
-bool common_batch::set_output(int32_t idx, bool value) {
-    if (idx < 0 || idx >= size()) {
-        return false;
-    }
-    tokens[idx].output = value;
-    return true;
-}
-
-bool common_batch::set_embd(int32_t idx, llama_embd embd) {
-    if (idx < 0 || idx >= size() || tokens[idx].embd.data != nullptr) {
-        return false;
-    }
-    tokens[idx].embd = embd;
-    return true;
-}
-
-int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
-    token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, {} };
-    for (int32_t j = 0; j < n_pos; ++j) {
-        t.pos[j] = pos[j];
-    }
-    tokens.push_back(t);
-    return size() - 1;
-}
-
-llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
-    GGML_ASSERT(batch && "common_batch was not initialized with a context");
-    GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size());
-
-    llama_batch_ext * res = batch.get();
-    llama_batch_ext_clear(res);
-
-    for (int32_t i = off; i < off + n; ++i) {
-        const token & t = tokens[i];
-
-        int32_t idx;
-        if (t.id != LLAMA_TOKEN_NULL) {
-            idx = llama_batch_ext_add_token(res, t.seq_id, t.id);
-            if (idx < 0) {
-                GGML_ABORT("%s: failed to add token %d at index %d (error %d, n = %d)\n", __func__, t.id, i, idx, n);
-            }
-            llama_batch_ext_set_pos(res, idx, t.pos.data());
-            if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
-                GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
-            }
-        } else {
-            idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
-            if (idx < 0) {
-                GGML_ABORT("%s: failed to add embedding at index %d (error %d, n = %d)\n", __func__, i, idx, n);
-            }
-            llama_batch_ext_set_pos(res, idx, t.pos.data());
-        }
-        GGML_ASSERT(idx == i - off);
-
-        for (const llama_seq_id seq_id : t.seq_ids_extra) {
-            if (!llama_batch_ext_add_seq(res, idx, seq_id)) {
-                GGML_ABORT("%s: failed to add seq %d to the entry at index %d\n", __func__, seq_id, i);
-            }
-        }
-        if (t.output) {
-            llama_batch_ext_set_output_logits(res, idx, true);
-        }
-    }
-
-    return res;
-}
-
-common_batch common_batch_get_one(llama_context * ctx, const llama_token * tokens, int32_t n_tokens) {
-    common_batch batch(ctx);
-
-    auto mem = llama_get_memory(ctx);
-    llama_pos pos = llama_memory_seq_pos_max(mem, 0) + 1; // -1 + 1 == 0 when the memory is empty
-
-    for (int32_t i = 0; i < n_tokens; ++i) {
-        const bool output = i == n_tokens - 1;
-        batch.add(tokens[i], pos, 0, output);
-        pos++;
-    }
-
-    return batch;
-}
-
-common_batch common_batch_get_one(llama_context * ctx, const llama_tokens & tokens) {
-    return common_batch_get_one(ctx, tokens.data(), (int32_t) tokens.size());
-}
-
 bool common_prompt_batch_decode(
               struct llama_context * ctx,
-                const llama_tokens & all_tokens,
+    const std::vector<llama_token> & all_tokens,
                                int   n_new,
                                int & n_past,
                                int   n_batch,
@@ -2225,9 +2222,7 @@ bool common_prompt_batch_decode(
         // Memory implementations in recurrent/hybrid models don't support removing tokens from their
         // memory, so we can't just remove the last token from the memory and replay the last token which
         // is the reason for this logic.
-        llama_tokens prefix_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_tokens_before_last);
-        common_batch batch_prefix = common_batch_get_one(ctx, prefix_tokens);
-        if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_prefix.get())) {
+        if (llama_decode(ctx, llama_batch_get_one(const_cast<llama_token*>(all_tokens.data() + offset), n_tokens_before_last))) {
             COM_ERR("%s", "failed to eval\n");
             return false;
         }
@@ -2236,18 +2231,18 @@ bool common_prompt_batch_decode(
         llama_state_save_file(ctx, state_path.data(), all_tokens.data(), all_tokens.size());
         COM_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size());
 
-        common_batch batch_last(ctx);
-        batch_last.add(all_tokens.back(), n_past, 0, true);
+        llama_token last_token = all_tokens.back();
+        llama_batch batch = llama_batch_get_one(&last_token, 1);
+        int32_t pos = n_past;
+        batch.pos = &pos;
 
-        if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch_last.get())) {
+        if (llama_decode(ctx, batch)) {
             COM_ERR("%s", "failed to eval last token\n");
             return false;
         }
         n_past++;
     } else {
-        llama_tokens new_tokens(all_tokens.begin() + offset, all_tokens.begin() + offset + n_new);
-        common_batch batch = common_batch_get_one(ctx, new_tokens);
-        if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
+        if (llama_decode(ctx, llama_batch_get_one(const_cast<llama_token*>(all_tokens.data() + offset), n_new))) {
             COM_ERR("%s", "failed to eval\n");
             return false;
         }

common/common.h

@@ -8,7 +8,6 @@
 #include "ggml.h"
 #include "llama.h"
 
-#include <array>
 #include <list>
 #include <set>
 #include <sstream>
@@ -17,7 +16,6 @@
 #include <vector>
 #include <map>
 #include <algorithm>
-#include <filesystem>
 #include <fstream>
 
 #if defined(_WIN32) && !defined(_WIN32_WINNT)
@@ -272,7 +270,7 @@ struct common_params_sampling {
         COMMON_SAMPLER_TYPE_TEMPERATURE,
     };
 
-    common_grammar                      grammar;          // optional grammar constraint (user / output-format / tool-calls)
+    common_grammar              grammar;      // optional grammar constraint (user / output-format / tool-calls)
     bool                                grammar_lazy = false;
     std::vector<common_grammar_trigger> grammar_triggers; // optional triggers (for lazy grammars)
     std::set<llama_token>               preserved_tokens;
@@ -485,7 +483,7 @@ struct common_params {
     enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
     enum llama_load_mode  load_mode  = LLAMA_LOAD_MODE_AUTO; // how to load the model
 
-    enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch
+    enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch
 
     common_cpu_params cpuparams;
     common_cpu_params cpuparams_batch;
@@ -633,10 +631,10 @@ struct common_params {
     int32_t checkpoint_min_step = 8192;  // minimum spacing between context checkpoints
     int32_t cache_ram_mib       = 8192;  // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
 
+    std::string hostname      = "127.0.0.1";
     std::string public_path   = "";                                                                         // NOLINT
     std::string api_prefix    = "";                                                                         // NOLINT
     std::string chat_template = "";                                                                         // NOLINT
-    std::vector<std::string> hostnames = {"127.0.0.1"};
     bool use_jinja = true;                                                                                  // NOLINT
 
     // server CORS params
@@ -659,7 +657,6 @@ struct common_params {
     std::string ssl_file_cert = "";                                                                         // NOLINT
 
     std::map<std::string, std::string> default_template_kwargs;
-    bool preserve_reasoning_specified = false;
 
     // CLI params
     std::string server_base; // if set, connect to this server instead of starting a new one
@@ -810,9 +807,7 @@ static std::vector<T> string_split(const std::string & str, char delim) {
     while (std::getline(str_stream, token, delim)) {
         T value;
         std::istringstream token_stream(token);
-        if (!(token_stream >> value)) {
-            throw std::invalid_argument("invalid value: \"" + token + "\"");
-        }
+        token_stream >> value;
         values.push_back(value);
     }
     return values;
@@ -880,21 +875,10 @@ void string_process_escapes(std::string & input);
 std::string string_from(bool value);
 std::string string_from(const std::vector<int> & values);
 std::string string_from(const struct llama_context * ctx, const std::vector<llama_token> & tokens);
+std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch);
 
 bool glob_match(const std::string & pattern, const std::string & str);
 
-//
-// Unicode utils
-//
-
-#ifdef _WIN32
-std::wstring utf8_to_wstring(const std::string & str);
-std::string  wstring_to_utf8(const std::wstring & str);
-#endif
-
-// returns the path as a UTF-8 string, preserving its separators
-std::string fs_path_to_utf8(const std::filesystem::path & path);
-
 //
 // Environment utils
 //
@@ -904,19 +888,17 @@ std::string fs_path_to_utf8(const std::filesystem::path & path);
 std::string common_get_env(const std::string & name);
 void        common_set_env(const std::string & name, const std::string & value);
 
-// reads a path from the environment, an unset variable gives an empty path
-std::filesystem::path common_get_path_from_env(const std::string & name);
-
 //
 // Filesystem utils
 //
 
 bool fs_validate_filename(const std::string & filename, bool allow_subdirs = false);
+bool fs_create_directory_with_parents(const std::string & path);
 bool fs_is_directory(const std::string & path);
 
-std::filesystem::path fs_get_cache_directory();
-std::filesystem::path fs_get_cache_file(const std::string & filename);
-std::filesystem::path fs_get_config_directory();
+std::string fs_get_cache_directory();
+std::string fs_get_cache_file(const std::string & filename);
+std::string fs_get_config_directory();
 
 struct common_file_info {
     std::string path;
@@ -929,8 +911,6 @@ std::vector<common_file_info> fs_list(const std::string & path, bool include_dir
 // fs open, also handle UTF8 on Windows
 std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode);
 
-void fs_write_atomic(const std::filesystem::path & path, const std::string & data);
-
 //
 // TTY utils
 //
@@ -1031,62 +1011,14 @@ struct common_memory {
 // Batch utils
 //
 
-// wrapper around llama_batch_ext that provide getter functions for downstream code
-// entries can exceed n_batch, use get_sub_batch() to decode them in chunks
-struct common_batch {
-    struct token {
-        llama_token  id;
-        std::array<llama_pos, GGML_MROPE_SECTIONS> pos; // only pos[0] is used for text tokens
-        llama_seq_id seq_id; // the first sequence id, see add_seq()
-        bool         output;
-        llama_embd   embd; // non-owning view of the data passed to add_embd()/set_embd(), data == NULL if none
-        std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
-    };
+void common_batch_clear(struct llama_batch & batch);
 
-    std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
-    llama_batch_ext_ptr batch;
-
-    int32_t n_pos = 1; // positions per embedding entry, GGML_MROPE_SECTIONS for MROPE/IMROPE
-
-    common_batch() = default;
-    common_batch(struct llama_context * ctx);
-
-    llama_batch_ext * get() { return get_sub_batch(0, size()); }
-
-    // render entries [off, off + n) into batch, the result is overwritten by the next call
-    llama_batch_ext * get_sub_batch(int32_t off, int32_t n);
-
-    // content type of the batch, all entries carry the same combination
-    bool has_token() const { return !tokens.empty() && tokens[0].id != LLAMA_TOKEN_NULL; }
-    bool has_embd () const { return !tokens.empty() && tokens[0].embd.data != nullptr; }
-
-    void clear();
-
-    // returns the batch index
-    int32_t add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output);
-
-    // same, with the entry shared by all seq_ids (must not be empty)
-    int32_t add(llama_token id, llama_pos pos, const std::vector<llama_seq_id> & seq_ids, bool output);
-
-    // add the entry at idx to another sequence, tokens[idx].seq_id keeps the first one
-    bool add_seq(int32_t idx, llama_seq_id seq_id);
-
-    bool set_output(int32_t idx, bool value);
-
-    // attach a token embedding to the entry at idx, can only be set once per entry
-    bool set_embd(int32_t idx, llama_embd embd);
-
-    // add an embedding-only entry (no token id)
-    // pos points to n_pos positions
-    int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
-
-    int32_t size() const { return (int32_t) tokens.size(); }
-};
-
-// create a single-sequence batch from a list of tokens
-// positions continue from the memory, last token always have output_logits set to true
-common_batch common_batch_get_one(struct llama_context * ctx, const llama_token * tokens, int32_t n_tokens);
-common_batch common_batch_get_one(struct llama_context * ctx, const llama_tokens & tokens);
+void common_batch_add(
+                 struct llama_batch & batch,
+                        llama_token   id,
+                          llama_pos   pos,
+    const std::vector<llama_seq_id> & seq_ids,
+                               bool   logits);
 
 // decodes a single batch of tokens for a prompt and manages session tokens
 //
@@ -1095,7 +1027,7 @@ common_batch common_batch_get_one(struct llama_context * ctx, const llama_tokens
 // tokens from memory, so this approach works across all model architectures.
 bool common_prompt_batch_decode(
               struct llama_context * ctx,
-                const llama_tokens & all_tokens,
+    const std::vector<llama_token> & all_tokens,
                                int   n_new,
                                int & n_past,
                                int   n_batch,

common/console.cpp

@@ -1,5 +1,4 @@
 #include "console.h"
-#include "common.h"
 #include "log.h"
 #include <vector>
 #include <iostream>
@@ -1019,7 +1018,6 @@ namespace console {
                     line.clear();
                     pop_cursor();
                 }
-                line += '\n';
                 has_more = false;
             }
         } else {
@@ -1051,10 +1049,13 @@ namespace console {
         if (!std::getline(std::wcin, wline)) {
             // Input stream is bad or EOF received
             line.clear();
+            GenerateConsoleCtrlEvent(CTRL_C_EVENT, 0);
             return false;
         }
 
-        line = wstring_to_utf8(wline);
+        int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), NULL, 0, NULL, NULL);
+        line.resize(size_needed);
+        WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), &line[0], size_needed, NULL, NULL);
 #else
         if (!std::getline(std::cin, line)) {
             // Input stream is bad or EOF received
@@ -1065,7 +1066,7 @@ namespace console {
         if (!line.empty()) {
             char last = line.back();
             if (last == '/') { // Always return control on '/' symbol
-                line.back() = '\n';
+                line.pop_back();
                 return false;
             }
             if (last == '\\') { // '\\' changes the default action

common/download.cpp

@@ -46,9 +46,39 @@
 // downloader
 //
 
+// validate repo name format: owner/repo
+static void write_file(const std::string & fname, const std::string & content) {
+    const std::string fname_tmp = fname + ".tmp";
+    std::ofstream     file(fname_tmp);
+    if (!file) {
+        throw std::runtime_error(string_format("error: failed to open file '%s'\n", fname.c_str()));
+    }
+
+    try {
+        file << content;
+        file.close();
+
+        // Makes write atomic
+        if (rename(fname_tmp.c_str(), fname.c_str()) != 0) {
+            LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, fname_tmp.c_str(), fname.c_str());
+            // If rename fails, try to delete the temporary file
+            if (remove(fname_tmp.c_str()) != 0) {
+                LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, fname_tmp.c_str());
+            }
+        }
+    } catch (...) {
+        // If anything fails, try to delete the temporary file
+        if (remove(fname_tmp.c_str()) != 0) {
+            LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, fname_tmp.c_str());
+        }
+
+        throw std::runtime_error(string_format("error: failed to write file '%s'\n", fname.c_str()));
+    }
+}
+
 static void write_etag(const std::string & path, const std::string & etag) {
     const std::string etag_path = path + ".etag";
-    fs_write_atomic(std::filesystem::u8path(etag_path), etag);
+    write_file(etag_path, etag);
     LOG_DBG("%s: file etag saved: %s\n", __func__, etag_path.c_str());
 }
 
@@ -244,12 +274,6 @@ static bool common_pull_file(httplib::Client & cli,
         return false;
     }
 
-    ofs.close();
-    if (!ofs) {
-        LOG_ERR("%s: error closing file: %s\n", __func__, path_tmp.c_str());
-        return false;
-    }
-
     return true;
 }
 
@@ -262,7 +286,7 @@ static int common_download_file_single_online(const std::string & url,
     static const int max_attempts        = 3;
     static const int retry_delay_seconds = 2;
 
-    const bool file_exists = std::filesystem::exists(std::filesystem::u8path(path));
+    const bool file_exists = std::filesystem::exists(path);
 
     if (file_exists && skip_etag) {
         LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
@@ -453,7 +477,7 @@ int common_download_file_single(const std::string & url,
         return common_download_file_single_online(url, path, online_opts, skip_etag);
     }
 
-    if (!std::filesystem::exists(std::filesystem::u8path(path))) {
+    if (!std::filesystem::exists(path)) {
         LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str());
         return -1;
     }
@@ -919,7 +943,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
         std::string model_filename = repo;
         std::replace(model_filename.begin(), model_filename.end(), '/', '_');
         model_filename += "_" + tag + ".gguf";
-        std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
+        std::string local_path = fs_get_cache_file(model_filename);
 
         const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
         common_download_opts opts;

common/fit.cpp

@@ -1,6 +1,5 @@
 #include "fit.h"
 
-#include "json.h"
 #include "log.h"
 
 #include "../src/llama-ext.h"
@@ -916,9 +915,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
 
     std::vector<std::array<std::string, 9>> table_data;
     table_data.reserve(devices.size());
-
-    // same data as the table below, for --log-jsonl consumers
-    common_json rows = common_json::array();
     const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
     const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
     const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
@@ -993,19 +989,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             std::to_string(mb.context / MiB),
             std::to_string(mb.compute / MiB),
             std::to_string(unaccounted / static_cast<int64_t>(MiB))});
-
-        rows.push_back({
-            {"kind",        "device"},
-            {"name",        name},
-            {"description", desc},
-            {"total",       total / MiB},
-            {"free",        free / MiB},
-            {"self",        self / MiB},
-            {"model",       mb.model / MiB},
-            {"context",     mb.context / MiB},
-            {"compute",     mb.compute / MiB},
-            {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
-        });
     }
 
     // print memory breakdown for host:
@@ -1021,15 +1004,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             std::to_string(mb_host.context / MiB),
             std::to_string(mb_host.compute / MiB),
             ""}); // unaccounted
-
-        rows.push_back({
-            {"kind",    "host"},
-            {"name",    "Host"},
-            {"self",    self / MiB},
-            {"model",   mb_host.model / MiB},
-            {"context", mb_host.context / MiB},
-            {"compute", mb_host.compute / MiB},
-        });
     }
 
     // print memory breakdown for all remaining buffer types:
@@ -1051,16 +1025,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             std::to_string(mb.context / MiB),
             std::to_string(mb.compute / MiB),
             ""}); // unaccounted
-
-        rows.push_back({
-            {"kind",    "buffer_type"},
-            {"name",    name},
-            {"self",    self / MiB},
-            {"model",   mb.model / MiB},
-            {"context", mb.context / MiB},
-            {"compute", mb.compute / MiB},
-        });
-
         seen_buffer_types.insert(buft);
     }
 
@@ -1078,11 +1042,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),
             td[6].c_str(), td[7].c_str(), td[8].c_str());
     }
-
-    LOG_JSON("fit_memory_breakdown", common_json({
-        {"unit", "MiB"},
-        {"rows", rows},
-    }));
 }
 
 void common_fit_print(

common/hf-cache.cpp

@@ -44,7 +44,8 @@ static fs::path get_cache_directory() {
             {HOME_DIR,                fs::path(".cache") / "huggingface" / "hub"}
         };
         for (const auto & entry : entries) {
-            if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) {
+            if (auto * p = std::getenv(entry.var); p && *p) {
+                fs::path base(p);
                 return entry.path.empty() ? base : base / entry.path;
             }
         }
@@ -61,10 +62,6 @@ static fs::path get_cache_directory() {
     return cache;
 }
 
-std::string get_cache_path() {
-    return fs_path_to_utf8(get_cache_directory());
-}
-
 static std::string folder_name_to_repo(const std::string & folder) {
     constexpr std::string_view prefix = "models--";
     if (folder.rfind(prefix, 0)) {
@@ -172,6 +169,28 @@ static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) {
     return b_end == b.end();
 }
 
+static void safe_write_file(const fs::path & path, const std::string & data) {
+    fs::path path_tmp = path.string() + ".tmp";
+
+    if (path.has_parent_path()) {
+        fs::create_directories(path.parent_path());
+    }
+
+    std::ofstream file(path_tmp);
+    file << data;
+    file.close();
+
+    std::error_code ec;
+
+    if (!file.fail()) {
+        fs::rename(path_tmp, path, ec);
+    }
+    if (file.fail() || ec) {
+        fs::remove(path_tmp, ec);
+        throw std::runtime_error("failed to write file: " + path.string());
+    }
+}
+
 static common_json api_get(const std::string & url,
                            const std::string & token) {
     auto [cli, parts] = common_http_client(url);
@@ -218,7 +237,6 @@ static std::string get_repo_commit(const std::string & repo_id,
         fs::path refs_path = get_repo_path(repo_id) / "refs";
         std::string name;
         std::string commit;
-        fs::path name_path;
 
         for (const auto & branch : json["branches"]) {
             if (!branch.is_object() ||
@@ -229,28 +247,24 @@ static std::string get_repo_commit(const std::string & repo_id,
             std::string _name = branch["name"].get<std::string>();
             std::string _commit = branch["targetCommit"].get<std::string>();
 
-            if (!is_valid_commit(_commit)) {
-                LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
+            if (!is_valid_subpath(refs_path, _name)) {
+                LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
                 continue;
             }
-            const fs::path candidate = fs::u8path(_name);
-
-            if (!is_valid_subpath(refs_path, candidate)) {
-                LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
+            if (!is_valid_commit(_commit)) {
+                LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
                 continue;
             }
 
             if (_name == "main") {
                 name = _name;
                 commit = _commit;
-                name_path = candidate;
                 break;
             }
 
             if (name.empty() || commit.empty()) {
                 name = _name;
                 commit = _commit;
-                name_path = candidate;
             }
         }
 
@@ -259,7 +273,7 @@ static std::string get_repo_commit(const std::string & repo_id,
             return {};
         }
 
-        fs_write_atomic(refs_path / name_path, commit);
+        safe_write_file(refs_path / name, commit);
         return commit;
 
     } catch (const common_json_error & e) {
@@ -308,9 +322,7 @@ hf_files get_repo_files(const std::string & repo_id,
             file.repo_id = repo_id;
             file.path = item["path"].get<std::string>();
 
-            const fs::path subpath = fs::u8path(file.path);
-
-            if (!is_valid_subpath(commit_path, subpath)) {
+            if (!is_valid_subpath(commit_path, file.path)) {
                 LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str());
                 continue;
             }
@@ -330,12 +342,12 @@ hf_files get_repo_files(const std::string & repo_id,
 
             file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
 
-            fs::path final_path = commit_path / subpath;
-            file.final_path = fs_path_to_utf8(final_path);
+            fs::path final_path = commit_path / file.path;
+            file.final_path = final_path.string();
 
             if (!file.oid.empty() && !fs::exists(final_path)) {
                 fs::path local_path = blobs_path / file.oid;
-                file.local_path = fs_path_to_utf8(local_path);
+                file.local_path = local_path.string();
             } else {
                 file.local_path = file.final_path;
             }
@@ -381,8 +393,8 @@ static std::string get_cached_ref(const fs::path & repo_path) {
 }
 
 hf_files get_cached_files(const std::string & repo_id) {
-    const fs::path cache_path = get_cache_directory();
-    if (!fs::exists(cache_path)) {
+    fs::path cache_dir = get_cache_directory();
+    if (!fs::exists(cache_dir)) {
         return {};
     }
 
@@ -393,7 +405,7 @@ hf_files get_cached_files(const std::string & repo_id) {
 
     hf_files files;
 
-    for (const auto & repo : fs::directory_iterator(cache_path)) {
+    for (const auto & repo : fs::directory_iterator(cache_dir)) {
         if (!repo.is_directory()) {
             continue;
         }
@@ -402,7 +414,7 @@ hf_files get_cached_files(const std::string & repo_id) {
         if (!fs::exists(snapshots_path)) {
             continue;
         }
-        std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
+        std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
 
         if (!is_valid_repo_id(_repo_id)) {
             continue;
@@ -425,9 +437,8 @@ hf_files get_cached_files(const std::string & repo_id) {
             if (!path.empty()) {
                 hf_file file;
                 file.repo_id = _repo_id;
-                const auto generic_path = path.generic_u8string();
-                file.path = std::string(generic_path.begin(), generic_path.end());
-                file.local_path = fs_path_to_utf8(entry.path());
+                file.path = path.generic_string();
+                file.local_path = entry.path().string();
                 file.final_path = file.local_path;
                 files.push_back(std::move(file));
             }
@@ -441,8 +452,8 @@ std::string finalize_file(const hf_file & file) {
     static std::atomic<bool> symlinks_disabled{false};
 
     std::error_code ec;
-    fs::path local_path = fs::u8path(file.local_path);
-    fs::path final_path = fs::u8path(file.final_path);
+    fs::path local_path(file.local_path);
+    fs::path final_path(file.final_path);
 
     if (local_path == final_path || fs::exists(final_path, ec)) {
         return file.final_path;
@@ -489,7 +500,7 @@ bool remove_cached_repo(const std::string & repo_id) {
     std::error_code ec;
     auto removed = fs::remove_all(repo_path, ec);
     if (ec) {
-        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
+        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
         return false;
     }
     return removed > 0;

common/hf-cache.h

@@ -32,7 +32,4 @@ std::string finalize_file(const hf_file & file);
 // Remove the entire cached directory for a repo, returns true if removed
 bool remove_cached_repo(const std::string & repo_id);
 
-// Returns the HuggingFace hub cache path
-std::string get_cache_path();
-
 } // namespace hf_cache

common/jinja/caps.cpp

@@ -117,7 +117,6 @@ caps caps_get(jinja::program & prog) {
 
     JJ_DEBUG("%s\n", ">>> Running capability check: typed content");
 
-    bool checks_for_string = false;
     static const std::string content_marker = "STRING_MARKER";
 
     // case: typed content support
@@ -137,10 +136,6 @@ caps caps_get(jinja::program & prog) {
         [&](context &, bool success, value & messages, value &, const std::string & rendered) {
             auto & content = messages->at(0)->at("content");
             caps_print_stats(content, "messages[0].content");
-            if (has_op(content, "test_is_string")) {
-                // checked if content is string
-                checks_for_string = true;
-            }
             bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
             if (used_as_array) {
                 // accessed as an array
@@ -156,33 +151,6 @@ caps caps_get(jinja::program & prog) {
         }
     );
 
-    if (checks_for_string) {
-        caps_try_execute(
-            prog,
-            [&]() {
-                // messages
-                return json::array({
-                    {
-                        {"role", "user"},
-                        {"content", json::array({
-                        })}
-                    }
-                });
-            },
-            nullptr, // ctx_fn
-            nullptr, // tools_fn
-            [&](context &, bool success, value & messages, value &, const std::string &) {
-                auto & content = messages->at(0)->at("content");
-                caps_print_stats(content, "messages[0].content");
-                bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
-                if (used_as_array && success) {
-                    // accessed as an array
-                    result.supports_typed_content = true;
-                }
-            }
-        );
-    }
-
     JJ_DEBUG("%s\n", ">>> Running capability check: system prompt");
 
     // case: system prompt support

common/jinja/parser.cpp

@@ -429,23 +429,15 @@ private:
             bool negate = false;
             if (is_identifier("not")) { ++current; negate = true; }
             auto test_id = parse_primary_expression();
-            if (is(token::open_paren)) {
-                test_id = parse_call_expression(std::move(test_id));
-            } else if (is(token::numeric_literal) || is(token::string_literal) || is(token::open_curly_bracket) || is(token::open_square_bracket) ||
-                    (is(token::identifier) && !is_identifier("and") && !is_identifier("or") && !is_identifier("else"))) {
-                size_t call_pos = current;
-                statements args;
-                args.push_back(parse_unary_expression());
-                test_id = mk_stmt<call_expression>(call_pos, std::move(test_id), std::move(args));
-            }
+            // FIXME: tests can also be expressed like this: if x is eq 3
+            if (is(token::open_paren)) test_id = parse_call_expression(std::move(test_id));
             operand = mk_stmt<test_expression>(start_pos, std::move(operand), negate, std::move(test_id));
         }
         return operand;
     }
 
     statement_ptr parse_filter_expression() {
-        // Filters/tests bind outside unary so -n|abs is (-n)|abs, not -(n|abs).
-        auto operand = parse_unary_expression();
+        auto operand = parse_call_member_expression();
         while (is(token::pipe)) {
             size_t start_pos = current;
             ++current; // consume pipe
@@ -456,15 +448,6 @@ private:
         return operand;
     }
 
-    statement_ptr parse_unary_expression() {
-        if (is(token::unary_operator)) {
-            size_t start_pos = current;
-            auto op = next();
-            return mk_stmt<unary_expression>(start_pos, op, parse_unary_expression());
-        }
-        return parse_call_member_expression();
-    }
-
     statement_ptr parse_call_member_expression() {
         // Handle member expressions recursively
         auto member = parse_member_expression(parse_primary_expression());

common/jinja/runtime.cpp

@@ -51,7 +51,7 @@ static void ensure_key_type_allowed(const value & val) {
 }
 
 // execute with error handling
-value statement::execute(context & ctx) const {
+value statement::execute(context & ctx) {
     try {
         return execute_impl(ctx);
     } catch (const continue_statement::signal & /* ex */) {
@@ -80,7 +80,7 @@ value statement::execute(context & ctx) const {
     }
 }
 
-value identifier::execute_impl(context & ctx) const {
+value identifier::execute_impl(context & ctx) {
     auto it = ctx.get_val(val);
     auto builtins = global_builtins();
     if (!it->is_undefined()) {
@@ -98,7 +98,7 @@ value identifier::execute_impl(context & ctx) const {
     }
 }
 
-value object_literal::execute_impl(context & ctx) const {
+value object_literal::execute_impl(context & ctx) {
     auto obj = mk_val<value_object>();
     for (const auto & pair : val) {
         value key = pair.first->execute(ctx);
@@ -109,7 +109,7 @@ value object_literal::execute_impl(context & ctx) const {
     return obj;
 }
 
-value binary_expression::execute_impl(context & ctx) const {
+value binary_expression::execute_impl(context & ctx) {
     value left_val = left->execute(ctx);
 
     // Logical operators
@@ -167,12 +167,6 @@ value binary_expression::execute_impl(context & ctx) const {
         }
         throw std::runtime_error("Cannot perform operation " + op.value + " on undefined values");
     } else if (is_val<value_none>(left_val) || is_val<value_none>(right_val)) {
-        if (!is_val<value_none>(right_val) && (op.value == "in" || op.value == "not in")) {
-            // case: none in {'low': 1}
-            // A null left operand is looked up like any other value.
-            bool member = test_is_in();
-            return mk_val<value_bool>(op.value == "in" ? member : !member);
-        }
         if (op.value == "+" || op.value == "~") {
             value res = mk_val<value_undefined>();
             if (workaround_concat_null_with_str(res)) {
@@ -317,7 +311,9 @@ static value try_builtin_func(context & ctx, const std::string & name, value & i
     throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
 }
 
-static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
+value filter_expression::execute_impl(context & ctx) {
+    value input = operand ? operand->execute(ctx) : val;
+
     JJ_DEBUG("Applying filter to %s", input->type().c_str());
 
     auto set_filter_alias = [](auto & filter_id) {
@@ -373,21 +369,22 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
     }
 }
 
-value filter_expression::execute_impl(context & ctx) const {
-    return apply_filter(ctx, filter, operand->execute(ctx));
-}
-
-value filter_statement::execute_impl(context & ctx) const {
+value filter_statement::execute_impl(context & ctx) {
     // eval body as string, then apply filter
     auto body_val = exec_statements(body, ctx);
     value_string parts = mk_val<value_string>();
     gather_string_parts_recursive(body_val, parts);
 
     JJ_DEBUG("FilterStatement: applying filter to body string of length %zu", parts->val_str.length());
-    return apply_filter(ctx, filter, parts);
+    filter_expression filter_expr(std::move(parts), std::move(filter));
+    value out = filter_expr.execute(ctx);
+
+    // this node can be reused later, make sure filter is preserved
+    this->filter = std::move(filter_expr.filter);
+    return out;
 }
 
-value test_expression::execute_impl(context & ctx) const {
+value test_expression::execute_impl(context & ctx) {
     // NOTE: "value is something" translates to function call "test_is_something(value)"
     const auto & builtins = global_builtins();
 
@@ -415,18 +412,12 @@ value test_expression::execute_impl(context & ctx) const {
         throw std::runtime_error("Invalid test expression");
     }
 
-    const std::string test_name = "test_is_" + test_id;
-    auto it = builtins.find(test_name);
-    JJ_DEBUG("Test expression %s '%s' %s (using function '%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_name.c_str());
+    auto it = builtins.find("test_is_" + test_id);
+    JJ_DEBUG("Test expression %s '%s' %s (using function 'test_is_%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_id.c_str());
     if (it == builtins.end()) {
         throw std::runtime_error("Unknown test '" + test_id + "'");
     }
 
-    if (ctx.is_get_stats) {
-        value_t::stats_t::mark_used(input);
-        input->stats.ops.insert(test_name);
-    }
-
     auto res = it->second(args);
 
     if (negate) {
@@ -436,7 +427,7 @@ value test_expression::execute_impl(context & ctx) const {
     }
 }
 
-value unary_expression::execute_impl(context & ctx) const {
+value unary_expression::execute_impl(context & ctx) {
     value operand_val = argument->execute(ctx);
     JJ_DEBUG("Executing unary expression with operator '%s'", op.value.c_str());
 
@@ -450,17 +441,12 @@ value unary_expression::execute_impl(context & ctx) const {
         } else {
             throw std::runtime_error("Unary - operator requires numeric operand");
         }
-    } else if (op.value == "+") {
-        if (is_val<value_int>(operand_val) || is_val<value_float>(operand_val)) {
-            return operand_val;
-        }
-        throw std::runtime_error("Unary + operator requires numeric operand");
     }
 
     throw std::runtime_error("Unknown unary operator '" + op.value + "'");
 }
 
-value if_statement::execute_impl(context & ctx) const {
+value if_statement::execute_impl(context & ctx) {
     value test_val = test->execute(ctx);
 
     auto out = mk_val<value_array>();
@@ -481,14 +467,20 @@ value if_statement::execute_impl(context & ctx) const {
     return str;
 }
 
-value for_statement::execute_impl(context & ctx) const {
+value for_statement::execute_impl(context & ctx) {
     context scope(ctx); // new scope for loop variables
 
-    const jinja::select_expression * select_expr = cast_stmt<select_expression>(iterable);
+    jinja::select_expression * select_expr = cast_stmt<select_expression>(iterable);
     statement_ptr test_expr_nullptr;
 
-    const statement_ptr & iter_expr = select_expr ? select_expr->lhs : iterable;
-    const statement_ptr & test_expr = select_expr ? select_expr->test : test_expr_nullptr;
+    statement_ptr & iter_expr = [&]() -> statement_ptr & {
+        auto tmp = cast_stmt<select_expression>(iterable);
+        return tmp ? tmp->lhs : iterable;
+    }();
+    statement_ptr & test_expr = [&]() -> statement_ptr & {
+        auto tmp = cast_stmt<select_expression>(iterable);
+        return tmp ? tmp->test : test_expr_nullptr;
+    }();
 
     JJ_DEBUG("Executing for statement, iterable type: %s", iter_expr->type().c_str());
 
@@ -537,6 +529,8 @@ value for_statement::execute_impl(context & ctx) const {
 
     std::vector<value> filtered_items;
     for (size_t i = 0; i < items.size(); ++i) {
+        context loop_scope(scope);
+
         value current = items[i];
 
         std::function<void(context&)> scope_update_fn = [](context &) { /* no-op */};
@@ -582,7 +576,6 @@ value for_statement::execute_impl(context & ctx) const {
         }
 
         if (select_expr && test_expr) {
-            context loop_scope(scope);
             scope_update_fn(loop_scope);
             value test_val = test_expr->execute(loop_scope);
             if (!test_val->as_bool()) {
@@ -640,7 +633,7 @@ value for_statement::execute_impl(context & ctx) const {
     return str;
 }
 
-value set_statement::execute_impl(context & ctx) const {
+value set_statement::execute_impl(context & ctx) {
     auto rhs = val ? val->execute(ctx) : exec_statements(body, ctx);
 
     if (is_stmt<identifier>(assignee)) {
@@ -739,7 +732,7 @@ static inline void bind_parameters(const std::string & name, const statements &
     }
 }
 
-value macro_statement::execute_impl(context & ctx) const {
+value macro_statement::execute_impl(context & ctx) {
     if (!is_stmt<identifier>(this->name)) {
         throw std::runtime_error("Macro name must be an identifier");
     }
@@ -762,7 +755,7 @@ value macro_statement::execute_impl(context & ctx) const {
     return mk_val<value_undefined>();
 }
 
-value call_statement::execute_impl(context & ctx) const {
+value call_statement::execute_impl(context & ctx) {
     auto call_expr = cast_stmt<call_expression>(this->call);
     if (!call_expr) {
         throw std::runtime_error("Call statement requires a valid call expression");
@@ -802,7 +795,7 @@ value call_statement::execute_impl(context & ctx) const {
     return callee_func->invoke(args);
 }
 
-value member_expression::execute_impl(context & ctx) const {
+value member_expression::execute_impl(context & ctx) {
     value object = this->object->execute(ctx);
 
     value property;
@@ -837,12 +830,6 @@ value member_expression::execute_impl(context & ctx) const {
         } else {
             property = this->property->execute(ctx);
         }
-    } else if (is_stmt<integer_literal>(this->property)) {
-        // syntax: obj.index
-        property = mk_val<value_int>(cast_stmt<integer_literal>(this->property)->val);
-        if (property->as_int() < 0) {
-            throw std::runtime_error("Static member property cannot be negative");
-        }
     } else {
         // syntax: obj.prop
         if (!is_stmt<identifier>(this->property)) {
@@ -887,7 +874,7 @@ value member_expression::execute_impl(context & ctx) const {
         JJ_DEBUG("Accessed property '%s' value, got type: %s", key.c_str(), val->type().c_str());
 
     } else if (is_val<value_array>(object) || is_val<value_string>(object)) {
-        if (is_val<value_int>(property) || is_val<value_bool>(property)) {
+        if (is_val<value_int>(property)) {
             int64_t index = property->as_int();
             JJ_DEBUG("Accessing %s index %d", object->type().c_str(), (int)index);
             if (is_val<value_array>(object)) {
@@ -910,6 +897,8 @@ value member_expression::execute_impl(context & ctx) const {
             JJ_DEBUG("Accessing %s built-in '%s'", is_val<value_array>(object) ? "array" : "string", key.c_str());
             val = try_builtin_func(ctx, key, object, true);
 
+        } else {
+            throw std::runtime_error("Cannot access property with non-string/non-number: got " + property->type());
         }
     } else {
         if (!is_val<value_string>(property)) {
@@ -923,17 +912,17 @@ value member_expression::execute_impl(context & ctx) const {
         value_t::stats_t::mark_used(val);
         value_t::stats_t::mark_used(object);
         value_t::stats_t::mark_used(property);
-        if (is_val<value_object>(object) || is_val<value_string>(property) || is_val<value_float>(property) || is_val<value_array>(property) || is_val<value_none>(property)) {
-            object->stats.ops.insert("object_access");
-        } else if (is_val<value_int>(property) || is_val<value_bool>(property)) {
+        if (is_val<value_int>(property)) {
             object->stats.ops.insert("array_access");
+        } else if (is_val<value_string>(property)) {
+            object->stats.ops.insert("object_access");
         }
     }
 
     return val;
 }
 
-value call_expression::execute_impl(context & ctx) const {
+value call_expression::execute_impl(context & ctx) {
     // gather arguments
     func_args args(ctx);
     for (auto & arg_stmt : this->args) {
@@ -951,7 +940,7 @@ value call_expression::execute_impl(context & ctx) const {
     return callee_func->invoke(args);
 }
 
-value keyword_argument_expression::execute_impl(context & ctx) const {
+value keyword_argument_expression::execute_impl(context & ctx) {
     if (!is_stmt<identifier>(key)) {
         throw std::runtime_error("Keyword argument key must be identifiers");
     }
@@ -975,7 +964,7 @@ std::string runtime::debug_dump_program(const program & prog, const std::string
         return std::string(lvl * 2, ' ');
     };
 
-    ctx.visitor = [&](bool is_leaf, const statement * node, std::vector<visitor_pair> children) {
+    ctx.visitor = [&](bool is_leaf, statement * node, std::vector<visitor_pair> children) {
         oss << indent(lvl) << node->type() << ":\n";
         lvl++;
         if (is_leaf) {

common/jinja/runtime.h

@@ -48,9 +48,9 @@ const T * cast_stmt(const statement_ptr & ptr) {
 void enable_debug(bool enable);
 
 // for visiting AST nodes
-// function signature: void(bool is_leaf, const statement * node, pair of <label, children>)
-using visitor_pair = std::pair<std::string, std::vector<const statement *>>;
-using visitor_fn = std::function<void(bool, const statement *, std::vector<visitor_pair>)>;
+// function signature: void(bool is_leaf, statement * node, pair of <label, children>)
+using visitor_pair = std::pair<std::string, std::vector<statement *>>;
+using visitor_fn = std::function<void(bool, statement *, std::vector<visitor_pair>)>;
 
 struct context {
     std::shared_ptr<std::string> src; // for debugging; use shared_ptr to avoid copying on scope creation
@@ -107,8 +107,8 @@ private:
 };
 
 // utils for visiting AST nodes
-static std::vector<const statement *> stmts_to_ptr(const statements & stmts) {
-    std::vector<const statement *> children;
+static std::vector<statement *> stmts_to_ptr(const statements & stmts) {
+    std::vector<statement *> children;
     for (const auto & stmt : stmts) {
         children.push_back(stmt.get());
     }
@@ -117,18 +117,17 @@ static std::vector<const statement *> stmts_to_ptr(const statements & stmts) {
 
 /**
  * Base class for all nodes in the AST.
- * The AST is shared between threads, so visit and execute must be const.
  */
 struct statement {
     size_t pos; // position in source, for debugging
     virtual ~statement() = default;
     virtual std::string type() const { return "Statement"; }
-    virtual void visit(context & ctx) const { ctx.visitor(true, this, {}); }
+    virtual void visit(context & ctx) { ctx.visitor(true, this, {}); }
 
     // execute_impl must be overridden by derived classes
-    virtual value execute_impl(context &) const { throw_exec_error(); }
+    virtual value execute_impl(context &) { throw_exec_error(); }
     // execute is the public method to execute a statement with error handling
-    value execute(context &) const;
+    value execute(context &);
 
 private:
     [[noreturn]] void throw_exec_error() const {
@@ -167,7 +166,7 @@ struct program : public statement {
     program() = default;
     explicit program(statements && body) : body(std::move(body)) {}
     std::string type() const override { return "Program"; }
-    [[noreturn]] value execute_impl(context &) const override {
+    [[noreturn]] value execute_impl(context &) override {
         throw std::runtime_error("Cannot execute program directly, use jinja::runtime instead");
     }
 };
@@ -183,8 +182,8 @@ struct if_statement : public statement {
     }
 
     std::string type() const override { return "If"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"test", {test.get()}},
             {"body", stmts_to_ptr(body)},
@@ -214,8 +213,8 @@ struct for_statement : public statement {
     }
 
     std::string type() const override { return "For"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"loopvar", {loopvar.get()}},
             {"iterable", {iterable.get()}},
@@ -234,7 +233,7 @@ struct break_statement : public statement {
         }
     };
 
-    [[noreturn]] value execute_impl(context &) const override {
+    [[noreturn]] value execute_impl(context &) override {
         throw break_statement::signal();
     }
 };
@@ -248,7 +247,7 @@ struct continue_statement : public statement {
         }
     };
 
-    [[noreturn]] value execute_impl(context &) const override {
+    [[noreturn]] value execute_impl(context &) override {
         throw continue_statement::signal();
     }
 };
@@ -256,7 +255,7 @@ struct continue_statement : public statement {
 // do nothing
 struct noop_statement : public statement {
     std::string type() const override { return "Noop"; }
-    value execute_impl(context &) const override {
+    value execute_impl(context &) override {
         return mk_val<value_undefined>();
     }
 };
@@ -273,8 +272,8 @@ struct set_statement : public statement {
     }
 
     std::string type() const override { return "Set"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"assignee", {assignee.get()}},
             {"value", {val.get()}},
@@ -295,8 +294,8 @@ struct macro_statement : public statement {
     }
 
     std::string type() const override { return "Macro"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"name", {name.get()}},
             {"args", stmts_to_ptr(args)},
@@ -309,7 +308,7 @@ struct comment_statement : public statement {
     std::string val;
     explicit comment_statement(const std::string & v) : val(v) {}
     std::string type() const override { return "Comment"; }
-    value execute_impl(context &) const override {
+    value execute_impl(context &) override {
         return mk_val<value_undefined>();
     }
 };
@@ -319,7 +318,7 @@ struct comment_statement : public statement {
 // Represents an omitted expression in a computed member, e.g. `a[]`.
 struct blank_expression : public expression {
     std::string type() const override { return "BlankExpression"; }
-    value execute_impl(context &) const override {
+    value execute_impl(context &) override {
         return mk_val<value_undefined>();
     }
 };
@@ -335,8 +334,8 @@ struct member_expression : public expression {
         chk_type<expression>(this->property);
     }
     std::string type() const override { return "MemberExpression"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"object", {object.get()}},
             {"property", {property.get()}}
@@ -354,8 +353,8 @@ struct call_expression : public expression {
         for (const auto& arg : this->args) chk_type<expression>(arg);
     }
     std::string type() const override { return "CallExpression"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"callee", {callee.get()}},
             {"args", stmts_to_ptr(args)}
@@ -370,7 +369,7 @@ struct identifier : public expression {
     std::string val;
     explicit identifier(const std::string & val) : val(val) {}
     std::string type() const override { return "Identifier"; }
-    value execute_impl(context & ctx) const override;
+    value execute_impl(context & ctx) override;
 };
 
 // Literals
@@ -379,7 +378,7 @@ struct integer_literal : public expression {
     int64_t val;
     explicit integer_literal(int64_t val) : val(val) {}
     std::string type() const override { return "IntegerLiteral"; }
-    value execute_impl(context &) const override {
+    value execute_impl(context &) override {
         return mk_val<value_int>(val);
     }
 };
@@ -388,7 +387,7 @@ struct float_literal : public expression {
     double val;
     explicit float_literal(double val) : val(val) {}
     std::string type() const override { return "FloatLiteral"; }
-    value execute_impl(context &) const override {
+    value execute_impl(context &) override {
         return mk_val<value_float>(val);
     }
 };
@@ -397,7 +396,7 @@ struct string_literal : public expression {
     std::string val;
     explicit string_literal(const std::string & val) : val(val) {}
     std::string type() const override { return "StringLiteral"; }
-    value execute_impl(context &) const override {
+    value execute_impl(context &) override {
         return mk_val<value_string>(val);
     }
 };
@@ -408,7 +407,7 @@ struct array_literal : public expression {
         for (const auto& item : this->val) chk_type<expression>(item);
     }
     std::string type() const override { return "ArrayLiteral"; }
-    value execute_impl(context & ctx) const override {
+    value execute_impl(context & ctx) override {
         auto arr = mk_val<value_array>();
         for (const auto & item_stmt : val) {
             arr->push_back(item_stmt->execute(ctx));
@@ -423,7 +422,7 @@ struct tuple_literal : public expression {
         for (const auto& item : this->val) chk_type<expression>(item);
     }
     std::string type() const override { return "TupleLiteral"; }
-    value execute_impl(context & ctx) const override {
+    value execute_impl(context & ctx) override {
         auto arr = mk_val<value_array>();
         for (const auto & item_stmt : val) {
             arr->push_back(item_stmt->execute(ctx));
@@ -442,7 +441,7 @@ struct object_literal : public expression {
         }
     }
     std::string type() const override { return "ObjectLiteral"; }
-    value execute_impl(context & ctx) const override;
+    value execute_impl(context & ctx) override;
 };
 
 // Complex Expressions
@@ -463,8 +462,8 @@ struct binary_expression : public expression {
         chk_type<expression>(this->right);
     }
     std::string type() const override { return "BinaryExpression"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"left", {left.get()}},
             {"right", {right.get()}}
@@ -477,7 +476,10 @@ struct binary_expression : public expression {
  * Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202
  */
 struct filter_expression : public expression {
+    // either an expression or a value is allowed
     statement_ptr operand;
+    value_string val; // will be set by filter_statement
+
     statement_ptr filter;
 
     filter_expression(statement_ptr && operand, statement_ptr && filter)
@@ -486,9 +488,14 @@ struct filter_expression : public expression {
         chk_type<identifier, call_expression>(this->filter);
     }
 
+    filter_expression(value_string && val, statement_ptr && filter)
+        : val(std::move(val)), filter(std::move(filter)) {
+        chk_type<identifier, call_expression>(this->filter);
+    }
+
     std::string type() const override { return "FilterExpression"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"operand", {operand.get()}},
             {"filter", {filter.get()}}
@@ -505,8 +512,8 @@ struct filter_statement : public statement {
         chk_type<identifier, call_expression>(this->filter);
     }
     std::string type() const override { return "FilterStatement"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"filter", {filter.get()}},
             {"body", stmts_to_ptr(body)}
@@ -530,14 +537,14 @@ struct select_expression : public expression {
         chk_type<expression>(this->test);
     }
     std::string type() const override { return "SelectExpression"; }
-    value execute_impl(context & ctx) const override {
+    value execute_impl(context & ctx) override {
         auto predicate = test->execute_impl(ctx);
         if (!predicate->as_bool()) {
             return mk_val<value_undefined>();
         }
         return lhs->execute_impl(ctx);
     }
-    void visit(context & ctx) const override {
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"lhs", {lhs.get()}},
             {"test", {test.get()}}
@@ -560,8 +567,8 @@ struct test_expression : public expression {
         chk_type<identifier, call_expression>(this->test);
     }
     std::string type() const override { return "TestExpression"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"operand", {operand.get()}},
             {"test", {test.get()}}
@@ -581,8 +588,8 @@ struct unary_expression : public expression {
         chk_type<expression>(this->argument);
     }
     std::string type() const override { return "UnaryExpression"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"argument", {argument.get()}}
         });
@@ -601,10 +608,10 @@ struct slice_expression : public expression {
         chk_type<expression>(this->step_expr);
     }
     std::string type() const override { return "SliceExpression"; }
-    [[noreturn]] value execute_impl(context &) const override {
+    [[noreturn]] value execute_impl(context &) override {
         throw std::runtime_error("must be handled by MemberExpression");
     }
-    void visit(context & ctx) const override {
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"start_expr", {start_expr.get()}},
             {"stop_expr", {stop_expr.get()}},
@@ -623,8 +630,8 @@ struct keyword_argument_expression : public expression {
         chk_type<expression>(this->val);
     }
     std::string type() const override { return "KeywordArgumentExpression"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"key", {key.get()}},
             {"val", {val.get()}}
@@ -638,7 +645,7 @@ struct spread_expression : public expression {
         chk_type<expression>(this->argument);
     }
     std::string type() const override { return "SpreadExpression"; }
-    void visit(context & ctx) const override {
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"argument", {argument.get()}}
         });
@@ -656,8 +663,8 @@ struct call_statement : public statement {
         for (const auto & arg : this->caller_args) chk_type<expression>(arg);
     }
     std::string type() const override { return "CallStatement"; }
-    value execute_impl(context & ctx) const override;
-    void visit(context & ctx) const override {
+    value execute_impl(context & ctx) override;
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"call", {call.get()}},
             {"caller_args", stmts_to_ptr(caller_args)},
@@ -678,7 +685,7 @@ struct ternary_expression : public expression {
         chk_type<expression>(this->false_expr);
     }
     std::string type() const override { return "Ternary"; }
-    value execute_impl(context & ctx) const override {
+    value execute_impl(context & ctx) override {
         value cond_val = condition->execute(ctx);
         if (cond_val->as_bool()) {
             return true_expr->execute(ctx);
@@ -686,7 +693,7 @@ struct ternary_expression : public expression {
             return false_expr->execute(ctx);
         }
     }
-    void visit(context & ctx) const override {
+    void visit(context & ctx) override {
         ctx.visitor(false, this, {
             {"condition", {condition.get()}},
             {"true_expr", {true_expr.get()}},

common/jinja/value.cpp

@@ -149,13 +149,6 @@ static value test_type_fn(const func_args & args) {
     JJ_DEBUG("test_type_fn: type=%s, %s or %s result=%d", typeid(T).name(), typeid(U).name(), typeid(V).name(), is_type ? 1 : 0);
     return mk_val<value_bool>(is_type);
 }
-template<typename T, typename U, typename V, typename W>
-static value test_type_fn(const func_args & args) {
-    args.ensure_count(1);
-    bool is_type = is_val<T>(args.get_pos(0)) || is_val<U>(args.get_pos(0)) || is_val<V>(args.get_pos(0)) || is_val<W>(args.get_pos(0));
-    JJ_DEBUG("test_type_fn: type=%s, %s, %s or %s result=%d", typeid(T).name(), typeid(U).name(), typeid(V).name(), typeid(W).name(), is_type ? 1 : 0);
-    return mk_val<value_bool>(is_type);
-}
 template<value_compare_op op>
 static value test_compare_fn(const func_args & args) {
     args.ensure_count(2, 2);
@@ -268,30 +261,6 @@ static value tojson(const func_args & args) {
     return mk_val<value_string>(json_str);
 }
 
-static value & get_attribute(const value & val, const value & attr, value & default_val) {
-    if (!attr->is_undefined()) {
-        if (is_val<value_array>(val)) {
-            value idx = attr;
-
-            if (is_val<value_string>(attr)) {
-                const std::string s = attr->as_string().str();
-                if (!s.empty() && std::all_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c); })) {
-                    try {
-                        idx = mk_val<value_int>(std::stoll(s));
-                    } catch (...) {
-                        idx = mk_val<value_undefined>();
-                    }
-                }
-            }
-
-            return val->at(idx, default_val);
-        } else if (is_val<value_object>(val)) {
-            return val->at(attr, default_val);
-        }
-    }
-    return default_val;
-}
-
 template<bool is_reject>
 static value selectattr(const func_args & args) {
     args.ensure_count(2, 4);
@@ -305,7 +274,10 @@ static value selectattr(const func_args & args) {
     if (args.count() == 2) {
         // example: array | selectattr("active")
         for (const auto & item : arr) {
-            value attr_val = get_attribute(item, attribute, val_default);
+            if (!is_val<value_object>(item)) {
+                throw raised_exception("selectattr: item is not an object");
+            }
+            value attr_val = item->at(attribute, val_default);
             bool is_selected = attr_val->as_bool();
             if constexpr (is_reject) is_selected = !is_selected;
             if (is_selected) out->push_back(item);
@@ -346,7 +318,10 @@ static value selectattr(const func_args & args) {
         }
         auto test_fn = it->second;
         for (const auto & item : arr) {
-            value attr_val = get_attribute(item, attribute, val_default);
+            if (!is_val<value_object>(item)) {
+                throw raised_exception("selectattr: item is not an object");
+            }
+            value attr_val = item->at(attribute, val_default);
             func_args test_args(args.ctx);
             test_args.push_back(attr_val); // attribute value
             test_args.push_back(extra_arg); // extra argument
@@ -373,43 +348,6 @@ static value default_value(const func_args & args) {
     return no_value ? args.get_pos(1) : args.get_pos(0);
 }
 
-static value toobject(const func_args & args) {
-    auto out = mk_val<value_object>();
-    value iter = args.get_pos(0, mk_val<value_undefined>());
-    bool iter_first = false;
-    if (is_val<value_array>(iter)) {
-        iter_first = true;
-        for (const auto & it : iter->as_array()) {
-            if (is_val<value_array>(it) && it->as_array().size() == 2) {
-                auto tuple = it->as_array();
-                auto key = tuple[0];
-                auto val = tuple[1];
-                JJ_DEBUG("namespace/dict: adding key '%s'", key->as_string().str().c_str());
-                out->insert(key, val);
-            } else {
-                throw raised_exception("namespace/dict() iterable argument must consist of tuples, not " + it->type());
-            }
-        }
-    } else if (is_val<value_object>(iter)) {
-        iter_first = true;
-        for (const auto & pair : iter->as_ordered_object()) {
-            JJ_DEBUG("namespace/dict: adding key '%s'", pair.first->as_string().str().c_str());
-            out->insert(pair.first, pair.second);
-        }
-    }
-    for (const auto & arg : args.get_args()) {
-        if (is_val<value_kwarg>(arg)) {
-            auto kwarg = cast_val<value_kwarg>(arg);
-            JJ_DEBUG("namespace/dict: adding key '%s'", kwarg->key.c_str());
-            out->insert(kwarg->key, kwarg->val);
-        } else if (!iter_first) {
-            throw raised_exception("namespace/dict() arguments must be kwargs, dict and/or iterable of tuples, not " + arg->type());
-        }
-        iter_first = false;
-    }
-    return out;
-}
-
 const func_builtins & global_builtins() {
     static const func_builtins builtins = {
         {"raise_exception", [](const func_args & args) -> value {
@@ -417,8 +355,18 @@ const func_builtins & global_builtins() {
             std::string msg = args.get_pos(0)->as_string().str();
             throw raised_exception("Jinja Exception: " + msg);
         }},
-        {"dict", toobject},
-        {"namespace", toobject},
+        {"namespace", [](const func_args & args) -> value {
+            auto out = mk_val<value_object>();
+            for (const auto & arg : args.get_args()) {
+                if (!is_val<value_kwarg>(arg)) {
+                    throw raised_exception("namespace() arguments must be kwargs");
+                }
+                auto kwarg = cast_val<value_kwarg>(arg);
+                JJ_DEBUG("namespace: adding key '%s'", kwarg->key.c_str());
+                out->insert(kwarg->key, kwarg->val);
+            }
+            return out;
+        }},
         {"strftime_now", [](const func_args & args) -> value {
             args.ensure_vals<value_string>();
             std::string format = args.get_pos(0)->as_string().str();
@@ -503,8 +451,8 @@ const func_builtins & global_builtins() {
         {"test_is_integer", test_type_fn<value_int>},
         {"test_is_float", test_type_fn<value_float>},
         {"test_is_number", test_type_fn<value_int, value_float>},
-        {"test_is_iterable", test_type_fn<value_object, value_array, value_string, value_undefined>},
-        {"test_is_sequence", test_type_fn<value_object, value_array, value_string, value_undefined>},
+        {"test_is_iterable", test_type_fn<value_array, value_string, value_undefined>},
+        {"test_is_sequence", test_type_fn<value_array, value_string, value_undefined>},
         {"test_is_mapping", test_type_fn<value_object>},
         {"test_is_lower", [](const func_args & args) -> value {
             args.ensure_vals<value_string>();
@@ -567,28 +515,8 @@ const func_builtins & global_builtins() {
         }},
         {"test_is_sameas", [](const func_args & args) -> value {
             // Check if an object points to the same memory address as another object
-            args.ensure_count(2);
-            auto a = args.get_pos(0);
-            auto b = args.get_pos(1);
-            bool res = false;
-            if (!is_val<value_undefined>(a) && !is_val<value_undefined>(b)) {
-                if (is_val<value_none>(a) && is_val<value_none>(b)) {
-                    res = true;
-                } else if (is_val<value_bool>(a) && is_val<value_bool>(b)) {
-                    if (a->as_bool() == b->as_bool()) {
-                        res = true;
-                    }
-                } else if (is_val<value_int>(a) && is_val<value_int>(b)) {
-                    const int64_t x = a->as_int();
-                    // Allow comparison within small-int cache range
-                    if (x >= -5 && x <= 256 && x == b->as_int()) {
-                        res = true;
-                    }
-                } else if (a == b) {
-                    res = true;
-                }
-            }
-            return mk_val<value_bool>(res);
+            (void)args;
+            throw not_implemented_exception("sameas test not implemented");
         }},
         {"test_is_escaped", [](const func_args & args) -> value {
             (void)args;
@@ -1093,14 +1021,22 @@ const func_builtins & value_array_t::get_builtins() const {
             }
             value val_delim = args.get_kwarg_or_pos("d",         1);
             value attribute = args.get_kwarg_or_pos("attribute", 2);
-            value undef = mk_val<value_undefined>();
             const auto & arr = args.get_pos(0)->as_array();
+            const bool attr_is_int = is_val<value_int>(attribute);
+            if (!attribute->is_undefined() && !is_val<value_string>(attribute) && !attr_is_int) {
+                throw raised_exception("join() attribute must be string or integer");
+            }
+            const int64_t attr_int = attr_is_int ? attribute->as_int() : 0;
             const std::string delim = val_delim->is_undefined() ? "" : val_delim->as_string().str();
             std::string result;
             for (size_t i = 0; i < arr.size(); ++i) {
                 value val_arr = arr[i];
                 if (!attribute->is_undefined()) {
-                    val_arr = get_attribute(val_arr, attribute, undef);
+                    if (attr_is_int && is_val<value_array>(val_arr)) {
+                        val_arr = val_arr->at(attr_int);
+                    } else if (!attr_is_int && is_val<value_object>(val_arr)) {
+                        val_arr = val_arr->at(attribute);
+                    }
                 }
                 if (!is_val<value_string>(val_arr) && !is_val<value_int>(val_arr) && !is_val<value_float>(val_arr)) {
                     throw raised_exception("join() can only join arrays of strings or numerics");
@@ -1132,11 +1068,21 @@ const func_builtins & value_array_t::get_builtins() const {
             }
             value val       = args.get_pos(0);
             value attribute = args.get_kwarg_or_pos("attribute", 1);
+            const bool attr_is_int = is_val<value_int>(attribute);
+            if (!is_val<value_string>(attribute) && !attr_is_int) {
+                throw raised_exception("map: attribute must be string or integer");
+            }
+            const int64_t attr_int = attr_is_int ? attribute->as_int() : 0;
             value default_val = args.get_kwarg("default", mk_val<value_undefined>());
             auto out = mk_val<value_array>();
             auto arr = val->as_array();
             for (const auto & item : arr) {
-                value attr_val = get_attribute(item, attribute, default_val);
+                value attr_val;
+                if (attr_is_int) {
+                    attr_val = is_val<value_array>(item) ? item->at(attr_int, default_val) : default_val;
+                } else {
+                    attr_val = is_val<value_object>(item) ? item->at(attribute, default_val) : default_val;
+                }
                 out->push_back(attr_val);
             }
             return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(out->as_array())) : out;
@@ -1173,14 +1119,22 @@ const func_builtins & value_array_t::get_builtins() const {
             // FIXME: sorting is currently always case sensitive
             //const bool case_sensitive = val_case->as_bool(); // undefined == false
             const bool reverse = val_reverse->as_bool(); // undefined == false
-            value undef = mk_val<value_undefined>();
+            const bool attr_is_int = is_val<value_int>(attribute);
+            const int64_t attr_int = attr_is_int ? attribute->as_int() : 0;
             std::vector<value> arr = val->as_array(); // copy
             std::sort(arr.begin(), arr.end(),[&](const value & a, const value & b) {
                 value val_a = a;
                 value val_b = b;
                 if (!attribute->is_undefined()) {
-                    val_a = get_attribute(a, attribute, undef);
-                    val_b = get_attribute(b, attribute, undef);
+                    if (attr_is_int && is_val<value_array>(a) && is_val<value_array>(b)) {
+                        val_a = a->at(attr_int);
+                        val_b = b->at(attr_int);
+                    } else if (!attr_is_int && is_val<value_object>(a) && is_val<value_object>(b)) {
+                        val_a = a->at(attribute);
+                        val_b = b->at(attribute);
+                    } else {
+                        throw raised_exception("sort: unsupported object attribute comparison between " + a->type() + " and " + b->type());
+                    }
                 }
                 return value_compare(val_a, val_b, reverse ? value_compare_op::gt : value_compare_op::lt);
             });
@@ -1198,23 +1152,19 @@ const func_builtins & value_array_t::get_builtins() const {
             args.ensure_vals<value_array>();
             value val_case    = args.get_kwarg_or_pos("case_sensitive", 1);
             value attribute   = args.get_kwarg_or_pos("attribute",      2);
+            if (!attribute->is_undefined()) {
+                throw not_implemented_exception("min: attribute not implemented");
+            }
             // FIXME: min is currently always case sensitive
             (void) val_case;
-            value undef = mk_val<value_undefined>();
             const auto & arr = args.get_pos(0)->as_array();
             if (arr.empty()) {
-                return undef;
+                return mk_val<value_undefined>();
             }
             value result = arr[0];
-            for (const auto & item : arr) {
-                value val_arr = item;
-                value val_cmp = result;
-                if (!attribute->is_undefined()) {
-                    val_arr = get_attribute(val_arr, attribute, undef);
-                    val_cmp = get_attribute(val_cmp, attribute, undef);
-                }
-                if (value_compare(val_arr, val_cmp, value_compare_op::lt)) {
-                    result = item;
+            for (size_t i = 1; i < arr.size(); ++i) {
+                if (value_compare(arr[i], result, value_compare_op::lt)) {
+                    result = arr[i];
                 }
             }
             return result;
@@ -1224,23 +1174,19 @@ const func_builtins & value_array_t::get_builtins() const {
             args.ensure_vals<value_array>();
             value val_case    = args.get_kwarg_or_pos("case_sensitive", 1);
             value attribute   = args.get_kwarg_or_pos("attribute",      2);
+            if (!attribute->is_undefined()) {
+                throw not_implemented_exception("max: attribute not implemented");
+            }
             // FIXME: max is currently always case sensitive
             (void) val_case;
-            value undef = mk_val<value_undefined>();
             const auto & arr = args.get_pos(0)->as_array();
             if (arr.empty()) {
-                return undef;
+                return mk_val<value_undefined>();
             }
             value result = arr[0];
-            for (const auto & item : arr) {
-                value val_arr = item;
-                value val_cmp = result;
-                if (!attribute->is_undefined()) {
-                    val_arr = get_attribute(val_arr, attribute, undef);
-                    val_cmp = get_attribute(val_cmp, attribute, undef);
-                }
-                if (value_compare(val_arr, val_cmp, value_compare_op::gt)) {
-                    result = item;
+            for (size_t i = 1; i < arr.size(); ++i) {
+                if (value_compare(arr[i], result, value_compare_op::gt)) {
+                    result = arr[i];
                 }
             }
             return result;

common/jinja/value.h

@@ -433,12 +433,6 @@ struct value_array_t : public value_t {
         }
         return val_arr[index];
     }
-    virtual value & at(const value & index, value & default_val) override {
-        if (!is_val<value_int>(index) && !is_val<value_bool>(index)) {
-            return default_val;
-        }
-        return at(index->as_int(), default_val);
-    }
     virtual const func_builtins & get_builtins() const override;
     virtual bool is_hashable() const override {
         if (std::all_of(val_arr.begin(), val_arr.end(), [&](auto & val) -> bool {

common/json-schema-to-grammar.cpp

@@ -1,7 +1,5 @@
 #include "json-schema-to-grammar.h"
 #include "common.h"
-#include "trie.h"
-#include "unicode.h"
 
 #include <algorithm>
 #include <limits>
@@ -321,8 +319,7 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
         case 'x': n_hex = 2; break;
         case 'u': n_hex = 4; break;
         case 'U': n_hex = 8; break;
-        // keep in sync with parse_char() in src/llama-grammar.cpp
-        case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']': case '-':
+        case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']':
             return 2;
         default:
             return 0;
@@ -339,20 +336,18 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
     return 2 + n_hex;
 }
 
-class common_chat_schema_converter {
+class common_schema_converter {
 private:
+    friend class common_schema_info;
     friend std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options);
+    std::function<json(const std::string &)> _fetch_json;
     bool _dotall;
     std::map<std::string, std::string> _rules;
+    std::unordered_map<std::string, json> _refs;
     std::unordered_set<std::string> _refs_being_resolved;
     std::vector<std::string> _errors;
     std::vector<std::string> _warnings;
 
-    template <typename T>
-    static const T & as(const common_chat_schema & node) {
-        return static_cast<const T &>(node);
-    }
-
     std::string _add_rule(const std::string & name, const std::string & rule) {
         std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
         if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
@@ -368,11 +363,11 @@ private:
         return key;
     }
 
-    std::string _generate_union_rule(const std::string & name, const std::vector<common_chat_schema_ptr> & alt_schemas) {
+    std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
         std::vector<std::string> rules;
         rules.reserve(alt_schemas.size());
         for (size_t i = 0; i < alt_schemas.size(); i++) {
-            rules.push_back(visit(*alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
+            rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
         }
         return string_join(rules, " | ");
     }
@@ -639,68 +634,85 @@ private:
             -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["]
     */
     std::string _not_strings(const std::vector<std::string> & strings) {
-        common_trie trie(strings);
+
+        struct TrieNode {
+            std::map<char, TrieNode> children;
+            bool is_end_of_string;
+
+            TrieNode() : is_end_of_string(false) {}
+
+            void insert(const std::string & string) {
+                auto *node = this;
+                for (char c : string) {
+                    node = &node->children[c];
+                }
+                node->is_end_of_string = true;
+            }
+        };
+
+        TrieNode trie;
+        for (const auto & s : strings) {
+            trie.insert(s);
+        }
 
         std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
         std::ostringstream out;
         out << "[\"] ( ";
-        std::function<void(size_t)> visit = [&](size_t idx) {
-            const auto & node = trie.nodes[idx];
-            std::string rejects;
+        std::function<void(const TrieNode &)> visit = [&](const TrieNode & node) {
+            std::ostringstream rejects;
             auto first = true;
-            for (const auto & [cpt, child] : node.children) {
-                std::string c = common_unicode_cpt_to_utf8(cpt);
-                rejects += c;
+            for (const auto & kv : node.children) {
+                rejects << kv.first;
                 if (first) {
                     first = false;
                 } else {
                     out << " | ";
                 }
-                out << "[" << c << "]";
-                if (!trie.nodes[child].children.empty()) {
+                out << "[" << kv.first << "]";
+                if (!kv.second.children.empty()) {
                     out << " (";
-                    visit(child);
+                    visit(kv.second);
                     out << ")";
-                } else {
+                } else if (kv.second.is_end_of_string) {
                     out << " " << char_rule << "+";
                 }
             }
             if (!node.children.empty()) {
-                out << " | [^\"" << rejects << "] " << char_rule << "*";
+                if (!first) {
+                    out << " | ";
+                }
+                out << "[^\"" << rejects.str() << "] " << char_rule << "*";
             }
         };
-        visit(0);
+        visit(trie);
 
         out << " )";
-        if (trie.nodes[0].pattern < 0) {
+        if (!trie.is_end_of_string) {
             out << "?";
         }
         out << " [\"]";
         return out.str();
     }
 
-    std::string _resolve_ref(const common_chat_schema_ref & schema) {
-        auto it = schema.ref.find('#');
-        std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref;
+    std::string _resolve_ref(const std::string & ref) {
+        auto it = ref.find('#');
+        std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref;
         static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
         std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-");
-        if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(schema.ref) == _refs_being_resolved.end()) {
-            if (!schema.target) {
-                _errors.push_back("Unresolved $ref " + schema.ref);
-                return "";
-            }
-            _refs_being_resolved.insert(schema.ref);
-            ref_name = visit(*schema.target, ref_name);
-            _refs_being_resolved.erase(schema.ref);
+        if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {
+            _refs_being_resolved.insert(ref);
+            json resolved = _refs[ref];
+            ref_name = visit(resolved, ref_name);
+            _refs_being_resolved.erase(ref);
         }
         return ref_name;
     }
 
     std::string _build_object_rule(
-        const std::vector<std::pair<std::string, const common_chat_schema *>> & properties,
+        const std::vector<std::pair<std::string, json>> & properties,
         const std::unordered_set<std::string> & required,
         const std::string & name,
-        const common_chat_schema * additional_properties)
+        const json & additional_properties)
     {
         std::vector<std::string> required_props;
         std::vector<std::string> optional_props;
@@ -710,7 +722,7 @@ private:
             const auto &prop_name = kv.first;
             const auto &prop_schema = kv.second;
 
-            std::string prop_rule_name = visit(*prop_schema, name + (name.empty() ? "" : "-") + prop_name);
+            std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
             prop_kv_rule_names[prop_name] = _add_rule(
                 name + (name.empty() ? "" : "-") + prop_name + "-kv",
                 format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
@@ -722,10 +734,10 @@ private:
             }
             prop_names.push_back(prop_name);
         }
-        if (additional_properties) {
+        if ((additional_properties.is_boolean() && additional_properties.get<bool>()) || additional_properties.is_object()) {
             std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
             std::string value_rule =
-                additional_properties->kind() != common_chat_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value")
+                additional_properties.is_object() ? visit(additional_properties, sub_name + "-value")
                 : _add_primitive("value", PRIMITIVE_RULES.at("value"));
 
             auto key_rule =
@@ -736,10 +748,6 @@ private:
             optional_props.push_back("*");
         }
 
-        if (required_props.empty() && optional_props.empty()) {
-            return "\"{\" space \"}\"";
-        }
-
         std::string rule = "\"{\" space ";
         for (size_t i = 0; i < required_props.size(); i++) {
             if (i > 0) {
@@ -813,163 +821,267 @@ private:
     }
 
 public:
-    explicit common_chat_schema_converter(bool dotall) : _dotall(dotall) {
+    common_schema_converter(
+        const std::function<json(const std::string &)> & fetch_json,
+        bool dotall)
+          : _fetch_json(fetch_json), _dotall(dotall)
+    {
         _rules["space"] = SPACE_RULE;
     }
 
-    std::string add_schema(const std::string & name, const common_chat_schema & schema) {
-        return visit(schema, name);
+    void resolve_refs(json & schema, const std::string & url) {
+        /*
+        * Resolves all $ref fields in the given schema, fetching any remote schemas,
+        * replacing each $ref with absolute reference URL and populates _refs with the
+        * respective referenced (sub)schema dictionaries.
+        */
+        std::function<void(json &)> visit_refs = [&](json & n) {
+            if (n.is_array()) {
+                for (auto & x : n) {
+                    visit_refs(x);
+                }
+            } else if (n.is_object()) {
+                if (n.contains("$ref")) {
+                    std::string ref = n["$ref"];
+                    if (_refs.find(ref) == _refs.end()) {
+                        json target;
+                        if (ref.find("https://") == 0) {
+                            std::string base_url = ref.substr(0, ref.find('#'));
+                            auto it = _refs.find(base_url);
+                            if (it != _refs.end()) {
+                                target = it->second;
+                            } else {
+                                // Fetch the referenced schema and resolve its refs
+                                auto referenced = _fetch_json(ref);
+                                resolve_refs(referenced, base_url);
+                                _refs[base_url] = referenced;
+                            }
+                            if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) {
+                                return;
+                            }
+                        } else if (ref.find("#/") == 0) {
+                            target = schema;
+                            n["$ref"] = url + ref;
+                            ref = url + ref;
+                        } else {
+                            _errors.push_back("Unsupported ref: " + ref);
+                            return;
+                        }
+                        std::string pointer = ref.substr(ref.find('#') + 1);
+                        std::vector<std::string> tokens = string_split(pointer, "/");
+                        for (size_t i = 1; i < tokens.size(); ++i) {
+                            const std::string& sel = tokens[i];
+                            if (target.is_object() && target.contains(sel)) {
+                                target = target[sel];
+                            } else if (target.is_array()) {
+                                size_t sel_index;
+                                try {
+                                    sel_index = std::stoull(sel);
+                                } catch (const std::invalid_argument & e) {
+                                    sel_index = target.size();
+                                }
+                                if (sel_index >= target.size()) {
+                                    _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
+                                    return;
+                                }
+                                target = target[sel_index];
+                            } else {
+                                _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
+                                return;
+                            }
+                        }
+                        _refs[ref] = target;
+                    }
+                } else {
+                    for (const auto & kv : n.items()) {
+                        visit_refs(kv.value());
+                    }
+                }
+            }
+        };
+
+        visit_refs(schema);
     }
 
     static std::string _generate_constant_rule(const json & value) {
         return format_literal(value.dump());
     }
 
-    std::string _visit_primitive(const std::string & rule_name, const std::string & type) {
-        return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type));
-    }
-
-    std::string _visit_all_of(const common_chat_schema_all_of & schema, const std::string & name, const std::string & rule_name) {
-        std::unordered_set<std::string> required;
-        std::vector<std::pair<std::string, const common_chat_schema *>> properties;
-        std::map<std::string, size_t> enum_values;
-        std::function<void(const common_chat_schema &, bool)> add_component = [&](const common_chat_schema & comp, bool is_required) {
-            if (comp.kind() == common_chat_schema::KIND_REF) {
-                if (const auto * target = as<common_chat_schema_ref>(comp).target) {
-                    add_component(*target, is_required);
-                }
-            } else if (comp.kind() == common_chat_schema::KIND_OBJECT) {
-                for (const auto & prop : as<common_chat_schema_object>(comp).properties) {
-                    properties.emplace_back(prop.name, prop.schema.get());
-                    if (is_required) {
-                        required.insert(prop.name);
-                    }
-                }
-            } else if (comp.kind() == common_chat_schema::KIND_ENUM) {
-                for (const auto & v : as<common_chat_schema_enum>(comp).values) {
-                    enum_values[_generate_constant_rule(v)] += 1;
-                }
-            }
-        };
-        for (const auto & child : schema.children) {
-            if (child->kind() == common_chat_schema::KIND_ANY_OF) {
-                for (const auto & alt : as<common_chat_schema_any_of>(*child).children) {
-                    add_component(*alt, false);
-                }
-            } else {
-                add_component(*child, true);
-            }
-        }
-        if (!enum_values.empty()) {
-            std::vector<std::string> enum_intersection;
-            for (const auto & p : enum_values) {
-                if (p.second == schema.children.size()) {
-                    enum_intersection.push_back(p.first);
-                }
-            }
-            if (!enum_intersection.empty()) {
-                return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
-            }
-        }
-        return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr));
-    }
-
-    std::string visit(const common_chat_schema & schema, const std::string & name) {
+    std::string visit(const json & schema, const std::string & name) {
+        json schema_type = schema.contains("type") ? schema["type"] : json();
+        std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : "";
         std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
-        std::string sub_name  = name + (name.empty() ? "" : "-");
 
-        switch (schema.kind()) {
-            case common_chat_schema::KIND_REF:
-                return _add_rule(rule_name, _resolve_ref(as<common_chat_schema_ref>(schema)));
-            case common_chat_schema::KIND_ANY_OF:
-                return _add_rule(rule_name, _generate_union_rule(name, as<common_chat_schema_any_of>(schema).children));
-            case common_chat_schema::KIND_ALL_OF:
-                return _visit_all_of(as<common_chat_schema_all_of>(schema), name, rule_name);
-            case common_chat_schema::KIND_CONST:
-                return _add_rule(rule_name, _generate_constant_rule(as<common_chat_schema_const>(schema).value));
-            case common_chat_schema::KIND_ENUM: {
-                std::vector<std::string> enum_values;
-                for (const auto & v : as<common_chat_schema_enum>(schema).values) {
-                    enum_values.push_back(_generate_constant_rule(v));
-                }
-                return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
+        if (schema.contains("$ref")) {
+            return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
+        }
+        if (schema.contains("oneOf") || schema.contains("anyOf")) {
+            const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf");
+            std::vector<json> alt_schemas;
+            for (const auto & alt : alts) {
+                alt_schemas.push_back(alt);
             }
-            case common_chat_schema::KIND_OBJECT: {
-                const auto & obj = as<common_chat_schema_object>(schema);
-                if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_chat_schema::KIND_ANY) {
-                    return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
-                }
-                std::vector<std::pair<std::string, const common_chat_schema *>> properties;
-                std::unordered_set<std::string> required;
-                for (const auto & prop : obj.properties) {
-                    properties.emplace_back(prop.name, prop.schema.get());
-                    if (prop.required) {
-                        required.insert(prop.name);
+            return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
+        }
+        if (schema_type.is_array()) {
+            std::vector<json> schema_types;
+            for (const auto & t : schema_type) {
+                json schema_copy(schema);
+                schema_copy["type"] = t;
+                schema_types.push_back(schema_copy);
+            }
+            return _add_rule(rule_name, _generate_union_rule(name, schema_types));
+        }
+        if (schema.contains("const")) {
+            return _add_rule(rule_name, _generate_constant_rule(schema["const"]));
+        }
+        if (schema.contains("enum")) {
+            std::vector<std::string> enum_values;
+            for (const auto & v : schema["enum"]) {
+                enum_values.push_back(_generate_constant_rule(v));
+            }
+            return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
+        }
+        if ((schema_type.is_null() || schema_type == "object")
+                && (schema.contains("properties") ||
+                    (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
+            std::unordered_set<std::string> required;
+            if (schema.contains("required") && schema["required"].is_array()) {
+                for (const auto & item : schema["required"]) {
+                    if (item.is_string()) {
+                        required.insert(item.get<std::string>());
                     }
                 }
-                return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get()));
             }
-            case common_chat_schema::KIND_TUPLE: {
-                const auto & items = as<common_chat_schema_tuple>(schema).items;
+            std::vector<std::pair<std::string, json>> properties;
+            if (schema.contains("properties")) {
+                for (const auto & prop : schema["properties"].items()) {
+                    properties.emplace_back(prop.key(), prop.value());
+                }
+            }
+            return _add_rule(rule_name,
+                _build_object_rule(
+                    properties, required, name,
+                    schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
+        }
+        if ((schema_type.is_null() || schema_type == "object" || schema_type == "string") && schema.contains("allOf")) {
+            std::unordered_set<std::string> required;
+            std::vector<std::pair<std::string, json>> properties;
+            std::map<std::string, size_t> enum_values;
+            const std::string& hybrid_name = name;
+            std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) {
+                if (comp_schema.contains("$ref")) {
+                    add_component(_refs[comp_schema["$ref"]], is_required);
+                } else if (comp_schema.contains("properties")) {
+                    for (const auto & prop : comp_schema["properties"].items()) {
+                        properties.emplace_back(prop.key(), prop.value());
+                        if (is_required) {
+                            required.insert(prop.key());
+                        }
+                    }
+                } else if (comp_schema.contains("enum")) {
+                    for (const auto & v : comp_schema["enum"]) {
+                        const auto rule = _generate_constant_rule(v);
+                        if (enum_values.find(rule) == enum_values.end()) {
+                            enum_values[rule] = 0;
+                        }
+                        enum_values[rule] += 1;
+                    }
+                } else {
+                  // todo warning
+                }
+            };
+            for (const auto & t : schema["allOf"]) {
+                if (t.contains("anyOf")) {
+                    for (const auto & tt : t["anyOf"]) {
+                        add_component(tt, false);
+                    }
+                } else {
+                    add_component(t, true);
+                }
+            }
+            if (!enum_values.empty()) {
+                std::vector<std::string> enum_intersection;
+                for (const auto & p : enum_values) {
+                    if (p.second == schema["allOf"].size()) {
+                        enum_intersection.push_back(p.first);
+                    }
+                }
+                if (!enum_intersection.empty()) {
+                    return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
+                }
+            }
+            return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
+        }
+        if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
+            json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
+            if (items.is_array()) {
                 std::string rule = "\"[\" space ";
                 for (size_t i = 0; i < items.size(); i++) {
                     if (i > 0) {
                         rule += " \",\" space ";
                     }
-                    rule += visit(*items[i], sub_name + "tuple-" + std::to_string(i));
+                    rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
                 }
                 rule += " space \"]\"";
                 return _add_rule(rule_name, rule);
             }
-            case common_chat_schema::KIND_ARRAY: {
-                const auto & arr = as<common_chat_schema_array>(schema);
-                if (arr.items->kind() == common_chat_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) {
-                    return _visit_primitive(rule_name, "array");
-                }
-                std::string item_rule_name = visit(*arr.items, sub_name + "item");
-                int max_items = arr.max_items < 0 ? std::numeric_limits<int>::max() : arr.max_items;
-                return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\"");
-            }
-            case common_chat_schema::KIND_STRING: {
-                const auto & str = as<common_chat_schema_string>(schema);
-                if (!str.pattern.empty()) {
-                    return _visit_pattern(str.pattern, rule_name);
-                }
-                if (str.format == common_chat_schema::FORMAT_UUID) {
-                    return _visit_primitive(rule_name, "uuid");
-                }
-                if (str.format != common_chat_schema::FORMAT_NONE) {
-                    std::string prim_name = std::string(str.format == common_chat_schema::FORMAT_DATE ? "date" : str.format == common_chat_schema::FORMAT_TIME ? "time" : "date-time") + "-string";
-                    return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
-                }
-                if (str.min_length > 0 || str.max_length >= 0) {
-                    std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
-                    int max_len = str.max_length < 0 ? std::numeric_limits<int>::max() : str.max_length;
-                    return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, str.min_length, max_len) + " \"\\\"\"");
-                }
-                return _visit_primitive(rule_name, "string");
-            }
-            case common_chat_schema::KIND_INTEGER: {
-                const auto & i = as<common_chat_schema_integer>(schema);
-                if (i.minimum == std::numeric_limits<int64_t>::min() && i.maximum == std::numeric_limits<int64_t>::max()) {
-                    return _visit_primitive(rule_name, "integer");
-                }
-                std::stringstream out;
-                out << "(";
-                build_min_max_int(i.minimum, i.maximum, out);
-                out << ")";
-                return _add_rule(rule_name, out.str());
-            }
-            case common_chat_schema::KIND_NUMBER:
-                return _visit_primitive(rule_name, "number");
-            case common_chat_schema::KIND_BOOLEAN:
-                return _visit_primitive(rule_name, "boolean");
-            case common_chat_schema::KIND_NULL:
-                return _visit_primitive(rule_name, "null");
-            case common_chat_schema::KIND_ANY:
-                return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
+            std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
+            int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
+            json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
+            int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
+
+            return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\"");
         }
-        return "";
+        if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
+            return _visit_pattern(schema["pattern"], rule_name);
+        }
+        if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
+            return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
+        }
+        if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
+            auto prim_name = schema_format + "-string";
+            return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
+        }
+        if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
+            std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
+            int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
+            int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
+            return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\"");
+        }
+        if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
+            int64_t min_value = std::numeric_limits<int64_t>::min();
+            int64_t max_value = std::numeric_limits<int64_t>::max();
+            if (schema.contains("minimum")) {
+                min_value = schema["minimum"].get<int64_t>();
+            } else if (schema.contains("exclusiveMinimum")) {
+                min_value = schema["exclusiveMinimum"].get<int64_t>() + 1;
+            }
+            if (schema.contains("maximum")) {
+                max_value = schema["maximum"].get<int64_t>();
+            } else if (schema.contains("exclusiveMaximum")) {
+                max_value = schema["exclusiveMaximum"].get<int64_t>() - 1;
+            }
+            std::stringstream out;
+            out << "(";
+            build_min_max_int(min_value, max_value, out);
+            out << ")";
+            return _add_rule(rule_name, out.str());
+        }
+        if (schema.empty() || schema_type == "object") {
+            return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
+        }
+        if (schema_type.is_null() && schema.is_object()) {
+            // No type constraint and no recognized structural keywords (e.g. {"description": "..."}).
+            // Per JSON Schema semantics this is equivalent to {} and accepts any value.
+            return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
+        }
+        if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
+            _errors.push_back("Unrecognized schema: " + schema.dump());
+            return "";
+        }
+        // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
+        return _add_primitive(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>()));
     }
 
     void check_errors() {
@@ -990,6 +1102,134 @@ public:
     }
 };
 
+// common_schema_info implementation (pimpl)
+
+common_schema_info::common_schema_info()
+    : impl_(std::make_unique<common_schema_converter>(
+        [](const std::string &) { return json(); },
+        false)) {}
+
+common_schema_info::~common_schema_info() = default;
+
+common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
+common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
+
+void common_schema_info::resolve_refs(common_json & schema) {
+    impl_->resolve_refs(schema, "");
+}
+
+// Determines if a JSON schema can resolve to a string type through any path.
+// Some models emit raw string values rather than JSON-encoded strings for string parameters.
+// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
+// true, allowing callers to handle the value as a raw string for simplicity.
+bool common_schema_info::resolves_to_string(const common_json & schema) {
+    std::unordered_set<std::string> visited_refs;
+
+    std::function<bool(const json &)> check = [&](const json & s) -> bool {
+        if (!s.is_object()) {
+            return false;
+        }
+
+        // Handle $ref
+        if (s.contains("$ref")) {
+            const std::string & ref = s["$ref"];
+            if (visited_refs.find(ref) != visited_refs.end()) {
+                // Circular reference, assume not a string to be safe
+                return false;
+            }
+            visited_refs.insert(ref);
+            auto it = impl_->_refs.find(ref);
+            if (it != impl_->_refs.end()) {
+                return check(it->second);
+            }
+            return false;
+        }
+
+        // Check type field
+        if (s.contains("type")) {
+            const json & schema_type = s["type"];
+            if (schema_type.is_string()) {
+                if (schema_type == "string") {
+                    return true;
+                }
+            } else if (schema_type.is_array()) {
+                // Type can be an array like ["string", "null"]
+                for (const auto & t : schema_type) {
+                    if (t == "string") {
+                        return true;
+                    }
+                }
+            }
+        }
+
+        // Check oneOf/anyOf - if any alternative can be a string
+        if (s.contains("oneOf")) {
+            for (const auto & alt : s["oneOf"]) {
+                if (check(alt)) {
+                    return true;
+                }
+            }
+        }
+        if (s.contains("anyOf")) {
+            for (const auto & alt : s["anyOf"]) {
+                if (check(alt)) {
+                    return true;
+                }
+            }
+        }
+
+        // Check allOf - all components must be compatible with string type
+        if (s.contains("allOf")) {
+            bool all_string = true;
+            for (const auto & component : s["allOf"]) {
+                if (!check(component)) {
+                    all_string = false;
+                    break;
+                }
+            }
+            if (all_string) {
+                return true;
+            }
+        }
+
+        // Check const - if the constant value is a string
+        if (s.contains("const")) {
+            if (s["const"].is_string()) {
+                return true;
+            }
+        }
+
+        // Check enum - if any enum value is a string
+        if (s.contains("enum")) {
+            for (const auto & val : s["enum"]) {
+                if (val.is_string()) {
+                    return true;
+                }
+            }
+        }
+
+        // String-specific keywords imply string type
+        if (s.contains("pattern") || s.contains("minLength") || s.contains("maxLength")) {
+            return true;
+        }
+
+        // Check format - many formats imply string
+        if (s.contains("format")) {
+            const std::string & fmt = s["format"];
+            if (fmt == "date" || fmt == "time" || fmt == "date-time" ||
+                fmt == "uri" || fmt == "email" || fmt == "hostname" ||
+                fmt == "ipv4" || fmt == "ipv6" || fmt == "uuid" ||
+                fmt.find("uuid") == 0) {
+                return true;
+            }
+        }
+
+        return false;
+    };
+
+    return check(schema);
+}
+
 std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) {
 #ifdef LLAMA_USE_LLGUIDANCE
     if (!force_gbnf) {
@@ -998,29 +1238,25 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf)
 #else
     (void)force_gbnf;
 #endif // LLAMA_USE_LLGUIDANCE
-    try {
-        return json_schema_to_grammar(common_chat_schema_from_json(schema));
-    } catch (const std::runtime_error & e) {
-        throw std::invalid_argument(std::string("JSON schema conversion failed:\n") + e.what());
-    }
-}
-
-std::string json_schema_to_grammar(const common_chat_schema_document & schema) {
-    common_chat_schema_converter converter(false);
-    converter.visit(*schema.root, "");
-    converter.check_errors();
-    return converter.format_grammar();
+    return build_grammar([&](const common_grammar_builder & callbacks) {
+        auto copy = schema;
+        callbacks.resolve_refs(copy);
+        callbacks.add_schema("", copy);
+    });
 }
 
 std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options) {
-    common_chat_schema_converter converter(options.dotall);
+    common_schema_converter converter([&](const std::string &) { return json(); }, options.dotall);
     common_grammar_builder builder {
         /* .add_rule = */ [&](const std::string & name, const std::string & rule) {
             return converter._add_rule(name, rule);
         },
-        /* .add_schema = */ [&](const std::string & name, const common_chat_schema & schema) {
-            return converter.add_schema(name == "root" ? "" : name, schema);
+        /* .add_schema = */ [&](const std::string & name, const common_json & schema) {
+            return converter.visit(schema, name == "root" ? "" : name);
         },
+        /* .resolve_refs = */ [&](common_json & schema) {
+            converter.resolve_refs(schema, "");
+        }
     };
     cb(builder);
     converter.check_errors();

common/json-schema-to-grammar.h

@@ -1,17 +1,37 @@
 #pragma once
 
-#include "json-schema.h"
 #include "json.h"
 
 #include <functional>
+#include <memory>
 #include <string>
 
-std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false);
-std::string json_schema_to_grammar(const common_chat_schema_document & schema);
+std::string json_schema_to_grammar(const common_json & schema,
+                                   bool force_gbnf = false);
+
+class common_schema_converter;
+
+// Probes a JSON schema to extract information about its structure and type constraints.
+class common_schema_info {
+    std::unique_ptr<common_schema_converter> impl_;
+
+  public:
+    common_schema_info();
+    ~common_schema_info();
+
+    common_schema_info(const common_schema_info &) = delete;
+    common_schema_info & operator=(const common_schema_info &) = delete;
+    common_schema_info(common_schema_info &&) noexcept;
+    common_schema_info & operator=(common_schema_info &&) noexcept;
+
+    void resolve_refs(common_json & schema);
+    bool resolves_to_string(const common_json & schema);
+};
 
 struct common_grammar_builder {
-    std::function<std::string(const std::string &, const std::string &)>    add_rule;
-    std::function<std::string(const std::string &, const common_chat_schema &)> add_schema;
+    std::function<std::string(const std::string &, const std::string &)> add_rule;
+    std::function<std::string(const std::string &, const common_json &)> add_schema;
+    std::function<void(common_json &)> resolve_refs;
 };
 
 struct common_grammar_options {

common/json-schema.cpp

@@ -1,514 +0,0 @@
-#include "json-schema.h"
-#include "common.h"
-
-#include <cmath>
-#include <map>
-#include <stdexcept>
-#include <string>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-class common_chat_schema_builder {
-    const common_json &      root_;
-    common_chat_schema_document & doc_;
-
-    // the targets built here, moved into doc_ once the whole schema is built
-    std::map<std::string, common_chat_schema_ptr> refs_;
-
-    // ref nodes get their target once every $ref is built, a cycle would otherwise need it too early
-    std::vector<common_chat_schema_ref *> pending_;
-
-    [[noreturn]] static void fail(const std::string & path, const std::string & msg) {
-        throw std::runtime_error("JSON schema error at " + path + ": " + msg);
-    }
-
-    static int get_count(const common_json & schema, const std::string & key, const std::string & path, int def) {
-        if (!schema.contains(key)) {
-            return def;
-        }
-        const common_json & value = schema.at(key);
-        if (!value.is_number_integer() || value.get<int>() < 0) {
-            fail(path, key + " must be a non-negative integer");
-        }
-        return value.get<int>();
-    }
-
-    // a fractional bound is rounded inwards, towards the integers it still admits
-    static int64_t get_bound(const common_json & schema, const std::string & key, const std::string & path, bool round_up) {
-        const common_json & value = schema.at(key);
-        if (value.is_number_integer()) {
-            return value.get<int64_t>();
-        }
-        if (!value.is_number()) {
-            fail(path, key + " must be a number");
-        }
-        double d = value.get<double>();
-        return (int64_t) (round_up ? std::ceil(d) : std::floor(d));
-    }
-
-    static common_chat_schema::string_format get_format(const common_json & schema, const std::string & path) {
-        if (!schema.contains("format")) {
-            return common_chat_schema::FORMAT_NONE;
-        }
-        const common_json & value = schema.at("format");
-        if (!value.is_string()) {
-            fail(path, "format must be a string");
-        }
-        std::string format = value.get<std::string>();
-        if (format == "date") {
-            return common_chat_schema::FORMAT_DATE;
-        }
-        if (format == "time") {
-            return common_chat_schema::FORMAT_TIME;
-        }
-        if (format == "date-time") {
-            return common_chat_schema::FORMAT_DATE_TIME;
-        }
-        if (format == "uuid" || (format.size() == 5 && format.compare(0, 4, "uuid") == 0 && format[4] >= '1' && format[4] <= '5')) {
-            return common_chat_schema::FORMAT_UUID;
-        }
-        return common_chat_schema::FORMAT_NONE;
-    }
-
-    const common_json & resolve_ref(const std::string & ref, const std::string & path) {
-        const common_json * target = &root_;
-        auto tokens = string_split(ref.substr(1), "/");
-        for (size_t i = 1; i < tokens.size(); i++) {
-            const std::string & sel = tokens[i];
-            if (target->is_object() && target->contains(sel)) {
-                target = &target->at(sel);
-            } else if (target->is_array()) {
-                size_t idx;
-                try {
-                    idx = std::stoull(sel);
-                } catch (const std::logic_error &) {
-                    idx = target->size();
-                }
-                if (idx >= target->size()) {
-                    fail(path, "cannot resolve $ref " + ref + ", " + sel + " is out of range");
-                }
-                target = &target->at(idx);
-            } else {
-                fail(path, "cannot resolve $ref " + ref + ", " + sel + " not found");
-            }
-        }
-        return *target;
-    }
-
-    common_chat_schema_ptr build_ref(const common_json & value, const std::string & path) {
-        if (!value.is_string()) {
-            fail(path, "$ref must be a string");
-        }
-        std::string ref = value.get<std::string>();
-        if (ref.compare(0, 2, "#/") != 0) {
-            fail(path, "unsupported $ref " + ref + ", only references into the same document are supported");
-        }
-        if (refs_.find(ref) == refs_.end()) {
-            // reserve the key first, so that a cycle back to this $ref stops here
-            refs_[ref] = nullptr;
-            refs_[ref] = build_node(resolve_ref(ref, path), ref);
-        }
-        auto node = std::make_unique<common_chat_schema_ref>(ref);
-        pending_.push_back(node.get());
-        return node;
-    }
-
-    template <typename T>
-    common_chat_schema_ptr build_alternatives(const common_json & alts, const std::string & path) {
-        if (!alts.is_array()) {
-            fail(path, "must be an array of schemas");
-        }
-        if (alts.empty()) {
-            fail(path, "must not be empty");
-        }
-        auto node = std::make_unique<T>();
-        size_t i = 0;
-        for (const auto & alt : alts) {
-            node->children.push_back(build_node(alt, path + "/" + std::to_string(i++)));
-        }
-        return node;
-    }
-
-    common_chat_schema_ptr build_object(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_object>();
-
-        std::unordered_set<std::string> required;
-        if (schema.contains("required") && schema.at("required").is_array()) {
-            for (const auto & name : schema.at("required")) {
-                if (name.is_string()) {
-                    required.insert(name.get<std::string>());
-                }
-            }
-        }
-
-        if (schema.contains("properties")) {
-            const common_json & properties = schema.at("properties");
-            if (!properties.is_object()) {
-                fail(path, "properties must be an object");
-            }
-            for (const auto & [name, prop] : properties.items()) {
-                node->properties.push_back({name, build_node(prop, path + "/properties/" + name), required.count(name) > 0});
-            }
-        }
-
-        if (schema.contains("additionalProperties")) {
-            const common_json & additional = schema.at("additionalProperties");
-            if (additional.is_boolean()) {
-                if (additional.get<bool>()) {
-                    node->additional_properties = std::make_unique<common_chat_schema_any>();
-                }
-            } else if (additional.is_object()) {
-                node->additional_properties = build_node(additional, path + "/additionalProperties");
-            } else {
-                fail(path, "additionalProperties must be a boolean or a schema");
-            }
-        } else if (!schema.contains("properties")) {
-            // {"type": "object"} on its own accepts any object
-            node->additional_properties = std::make_unique<common_chat_schema_any>();
-        }
-
-        return node;
-    }
-
-    common_chat_schema_ptr build_array(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_array>();
-        if (schema.contains("items") || schema.contains("prefixItems")) {
-            // "items" wins when both are present; as in the converter, a schema instead of an array is the item schema
-            const std::string key = schema.contains("items") ? "items" : "prefixItems";
-            const common_json & items = schema.at(key);
-            if (items.is_array()) {
-                auto tuple = std::make_unique<common_chat_schema_tuple>();
-                size_t i = 0;
-                for (const auto & item : items) {
-                    tuple->items.push_back(build_node(item, path + "/" + key + "/" + std::to_string(i++)));
-                }
-                return tuple;
-            }
-            node->items = build_node(items, path + "/" + key);
-        } else {
-            node->items = std::make_unique<common_chat_schema_any>();
-        }
-        node->min_items = get_count(schema, "minItems", path, 0);
-        node->max_items = get_count(schema, "maxItems", path, -1);
-        return node;
-    }
-
-    common_chat_schema_ptr build_string(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_string>();
-        if (schema.contains("pattern")) {
-            const common_json & pattern = schema.at("pattern");
-            if (!pattern.is_string()) {
-                fail(path, "pattern must be a string");
-            }
-            node->pattern = pattern.get<std::string>();
-        }
-        node->format     = get_format(schema, path);
-        node->min_length = get_count(schema, "minLength", path, 0);
-        node->max_length = get_count(schema, "maxLength", path, -1);
-        return node;
-    }
-
-    common_chat_schema_ptr build_integer(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_integer>();
-        if (schema.contains("minimum")) {
-            node->minimum = get_bound(schema, "minimum", path, /* round_up */ true);
-        } else if (schema.contains("exclusiveMinimum")) {
-            node->minimum = get_bound(schema, "exclusiveMinimum", path, /* round_up */ false) + 1;
-        }
-        if (schema.contains("maximum")) {
-            node->maximum = get_bound(schema, "maximum", path, /* round_up */ false);
-        } else if (schema.contains("exclusiveMaximum")) {
-            node->maximum = get_bound(schema, "exclusiveMaximum", path, /* round_up */ true) - 1;
-        }
-        return node;
-    }
-
-    common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) {
-        if (!schema.is_object()) {
-            fail(path, "schema must be an object");
-        }
-        if (schema.contains("$ref")) {
-            return build_ref(schema.at("$ref"), path);
-        }
-        if (schema.contains("oneOf") || schema.contains("anyOf")) {
-            const std::string key = schema.contains("oneOf") ? "oneOf" : "anyOf";
-            return build_alternatives<common_chat_schema_any_of>(schema.at(key), path + "/" + key);
-        }
-
-        common_json type;
-        if (schema.contains("type")) {
-            type = schema.at("type");
-        }
-        if (type.is_array()) {
-            // {"type": ["a", "b"], ...} is {"anyOf": [{"type": "a", ...}, {"type": "b", ...}]}
-            if (type.empty()) {
-                fail(path, "type must not be empty");
-            }
-            auto node = std::make_unique<common_chat_schema_any_of>();
-            size_t i = 0;
-            for (const auto & t : type) {
-                common_json alt = schema;
-                alt["type"] = t;
-                node->children.push_back(build_node(alt, path + "/type/" + std::to_string(i++)));
-            }
-            return node;
-        }
-        if (schema.contains("const")) {
-            return std::make_unique<common_chat_schema_const>(schema.at("const"));
-        }
-        if (schema.contains("enum")) {
-            const common_json & values = schema.at("enum");
-            if (!values.is_array() || values.empty()) {
-                fail(path, "enum must be a non-empty array");
-            }
-            auto node = std::make_unique<common_chat_schema_enum>();
-            for (const auto & value : values) {
-                node->values.push_back(value);
-            }
-            return node;
-        }
-        if (!type.is_null() && !type.is_string()) {
-            fail(path, "type must be a string or an array of strings");
-        }
-
-        const std::string type_name = type.is_string() ? type.get<std::string>() : "";
-        const bool has_properties = schema.contains("properties") ||
-            (schema.contains("additionalProperties") && schema.at("additionalProperties") != true);
-
-        if (type_name.empty()) {
-            // without a type the structural keywords decide, in the same order as the converter
-            if (has_properties) {
-                return build_object(schema, path);
-            }
-            if (schema.contains("allOf")) {
-                return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
-            }
-            if (schema.contains("items") || schema.contains("prefixItems")) {
-                return build_array(schema, path);
-            }
-            if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_chat_schema::FORMAT_NONE) {
-                return build_string(schema, path);
-            }
-            return std::make_unique<common_chat_schema_any>();
-        }
-        if (type_name == "object") {
-            if (!has_properties && schema.contains("allOf")) {
-                return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
-            }
-            return build_object(schema, path);
-        }
-        if (type_name == "string") {
-            if (schema.contains("allOf")) {
-                return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
-            }
-            return build_string(schema, path);
-        }
-        if (type_name == "array") {
-            return build_array(schema, path);
-        }
-        if (type_name == "integer") {
-            return build_integer(schema, path);
-        }
-        if (type_name == "number") {
-            return std::make_unique<common_chat_schema_number>();
-        }
-        if (type_name == "boolean") {
-            return std::make_unique<common_chat_schema_boolean>();
-        }
-        if (type_name == "null") {
-            return std::make_unique<common_chat_schema_null>();
-        }
-        fail(path, "unrecognized type " + type_name);
-    }
-
-  public:
-    common_chat_schema_builder(const common_json & root, common_chat_schema_document & doc) : root_(root), doc_(doc) {}
-
-    common_chat_schema_ptr build() {
-        auto node = build_node(root_, "#");
-        for (auto & entry : refs_) {
-            doc_.refs[entry.first] = std::move(entry.second);
-        }
-        for (auto * ref : pending_) {
-            ref->target = doc_.refs.at(ref->ref).get();
-        }
-        return node;
-    }
-};
-
-common_chat_schema_document common_chat_schema_from_json(const common_json & schema) {
-    common_chat_schema_document doc;
-    doc.root = common_chat_schema_builder(schema, doc).build();
-    return doc;
-}
-
-static common_chat_schema::value_type json_type(const common_json & value) {
-    if (value.is_null()) {
-        return common_chat_schema::TYPE_NULL;
-    }
-    if (value.is_boolean()) {
-        return common_chat_schema::TYPE_BOOLEAN;
-    }
-    if (value.is_number_integer()) {
-        return common_chat_schema::TYPE_INTEGER;
-    }
-    if (value.is_number()) {
-        return common_chat_schema::TYPE_NUMBER;
-    }
-    if (value.is_string()) {
-        return common_chat_schema::TYPE_STRING;
-    }
-    if (value.is_array()) {
-        return common_chat_schema::TYPE_ARRAY;
-    }
-    return common_chat_schema::TYPE_OBJECT;
-}
-
-static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
-    switch (s.kind()) {
-        case common_chat_schema::KIND_ANY:
-            return common_chat_schema::type_set::all();
-        case common_chat_schema::KIND_NULL:
-            return { common_chat_schema::TYPE_NULL };
-        case common_chat_schema::KIND_BOOLEAN:
-            return { common_chat_schema::TYPE_BOOLEAN };
-        case common_chat_schema::KIND_NUMBER:
-            return { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER };
-        case common_chat_schema::KIND_INTEGER:
-            return { common_chat_schema::TYPE_INTEGER };
-        case common_chat_schema::KIND_STRING:
-            return { common_chat_schema::TYPE_STRING };
-        case common_chat_schema::KIND_ARRAY:
-        case common_chat_schema::KIND_TUPLE:
-            return { common_chat_schema::TYPE_ARRAY };
-        case common_chat_schema::KIND_OBJECT:
-            return { common_chat_schema::TYPE_OBJECT };
-        case common_chat_schema::KIND_CONST:
-            return { json_type(static_cast<const common_chat_schema_const &>(s).value) };
-        case common_chat_schema::KIND_ENUM: {
-            common_chat_schema::type_set types;
-            for (const auto & value : static_cast<const common_chat_schema_enum &>(s).values) {
-                types.add(json_type(value));
-            }
-            return types;
-        }
-        case common_chat_schema::KIND_REF: {
-            const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
-            if (!target || !visited.insert(target).second) {
-                // a cycle contributes no type, to be safe
-                return {};
-            }
-            auto types = value_types_impl(*target, visited);
-            visited.erase(target);
-            return types;
-        }
-        case common_chat_schema::KIND_ANY_OF: {
-            common_chat_schema::type_set types;
-            for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                types |= value_types_impl(*child, visited);
-            }
-            return types;
-        }
-        case common_chat_schema::KIND_ALL_OF: {
-            auto types = common_chat_schema::type_set::all();
-            for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
-                types &= value_types_impl(*child, visited);
-            }
-            return types;
-        }
-    }
-    return {};
-}
-
-common_chat_schema::type_set common_chat_schema::value_types() const {
-    std::unordered_set<const common_chat_schema *> visited;
-    return value_types_impl(*this, visited);
-}
-
-static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
-    switch (s.kind()) {
-        case common_chat_schema::KIND_STRING:
-            return true;
-        case common_chat_schema::KIND_CONST:
-            return static_cast<const common_chat_schema_const &>(s).value.is_string();
-        case common_chat_schema::KIND_ENUM:
-            for (const auto & v : static_cast<const common_chat_schema_enum &>(s).values) {
-                if (v.is_string()) {
-                    return true;
-                }
-            }
-            return false;
-        case common_chat_schema::KIND_REF: {
-            // a cycle is taken as not a string, to be safe
-            const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
-            if (!target || !visited.insert(target).second) {
-                return false;
-            }
-            bool result = may_be_string_impl(*target, visited);
-            visited.erase(target);
-            return result;
-        }
-        case common_chat_schema::KIND_ANY_OF:
-            for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                if (may_be_string_impl(*child, visited)) {
-                    return true;
-                }
-            }
-            return false;
-        case common_chat_schema::KIND_ALL_OF: {
-            // every child must allow a string, an any child constrains nothing
-            bool any_string = false;
-            for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
-                if (child->kind() == common_chat_schema::KIND_ANY) {
-                    continue;
-                }
-                if (!may_be_string_impl(*child, visited)) {
-                    return false;
-                }
-                any_string = true;
-            }
-            return any_string;
-        }
-        default:
-            return false;
-    }
-}
-
-bool common_chat_schema::may_be_string() const {
-    std::unordered_set<const common_chat_schema *> visited;
-    return may_be_string_impl(*this, visited);
-}
-
-const char * common_chat_schema::kind_name(node_kind kind) {
-    switch (kind) {
-        case KIND_ANY:     return "any";
-        case KIND_REF:     return "ref";
-        case KIND_ANY_OF:  return "anyOf";
-        case KIND_ALL_OF:  return "allOf";
-        case KIND_CONST:   return "const";
-        case KIND_ENUM:    return "enum";
-        case KIND_NULL:    return "null";
-        case KIND_BOOLEAN: return "boolean";
-        case KIND_NUMBER:  return "number";
-        case KIND_INTEGER: return "integer";
-        case KIND_STRING:  return "string";
-        case KIND_ARRAY:   return "array";
-        case KIND_TUPLE:   return "tuple";
-        case KIND_OBJECT:  return "object";
-    }
-    return "?";
-}
-
-const char * common_chat_schema::type_name(value_type type) {
-    switch (type) {
-        case TYPE_NULL:    return "null";
-        case TYPE_BOOLEAN: return "boolean";
-        case TYPE_NUMBER:  return "number";
-        case TYPE_INTEGER: return "integer";
-        case TYPE_STRING:  return "string";
-        case TYPE_ARRAY:   return "array";
-        case TYPE_OBJECT:  return "object";
-    }
-    return "?";
-}

common/json-schema.h

@@ -1,198 +0,0 @@
-#pragma once
-
-#include "json.h"
-
-#include <cstdint>
-#include <initializer_list>
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-// JSON schema, covering the subset that json_schema_to_grammar() can convert.
-
-struct common_chat_schema {
-    enum node_kind {
-        KIND_ANY,
-        KIND_REF,
-        KIND_ANY_OF,
-        KIND_ALL_OF,
-        KIND_CONST,
-        KIND_ENUM,
-        KIND_NULL,
-        KIND_BOOLEAN,
-        KIND_NUMBER,
-        KIND_INTEGER,
-        KIND_STRING,
-        KIND_ARRAY,
-        KIND_TUPLE,
-        KIND_OBJECT,
-    };
-
-    enum value_type {
-        TYPE_NULL,
-        TYPE_BOOLEAN,
-        TYPE_NUMBER,
-        TYPE_INTEGER,
-        TYPE_STRING,
-        TYPE_ARRAY,
-        TYPE_OBJECT,
-    };
-
-    enum string_format {
-        FORMAT_NONE,
-        FORMAT_UUID,  // uuid, uuid1 .. uuid5
-        FORMAT_DATE,
-        FORMAT_TIME,
-        FORMAT_DATE_TIME,
-    };
-
-    class type_set {
-        uint32_t mask_ = 0;
-
-      public:
-        type_set() = default;
-        type_set(std::initializer_list<value_type> types) {
-            for (auto type : types) {
-                add(type);
-            }
-        }
-
-        static type_set all() {
-            return { TYPE_NULL, TYPE_BOOLEAN, TYPE_NUMBER, TYPE_INTEGER, TYPE_STRING, TYPE_ARRAY, TYPE_OBJECT };
-        }
-
-        void add(value_type type) { mask_ |= 1u << type; }
-
-        bool has(value_type type) const { return (mask_ & (1u << type)) != 0; }
-        bool is_only(value_type type) const { return mask_ == (1u << type); }
-        bool empty() const { return mask_ == 0; }
-
-        type_set & operator|=(const type_set & other) { mask_ |= other.mask_; return *this; }
-        type_set & operator&=(const type_set & other) { mask_ &= other.mask_; return *this; }
-
-        bool operator==(const type_set & other) const { return mask_ == other.mask_; }
-        bool operator!=(const type_set & other) const { return mask_ != other.mask_; }
-    };
-
-    virtual ~common_chat_schema() = default;
-    virtual node_kind kind() const = 0;
-
-    type_set value_types() const;
-
-    // Whether a value matching the schema may be a string, through any branch of it.
-    bool may_be_string() const;
-
-    static const char * kind_name(node_kind kind);
-    static const char * type_name(value_type type);
-};
-
-using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>;
-
-struct common_chat_schema_any : common_chat_schema {
-    node_kind kind() const override { return KIND_ANY; }
-};
-
-// {"$ref": "#/..."}, only references into the same document are supported
-struct common_chat_schema_ref : common_chat_schema {
-    std::string                ref;
-    const common_chat_schema * target = nullptr;  // owned by common_chat_schema_document::refs
-
-    explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {}
-
-    node_kind kind() const override { return KIND_REF; }
-};
-
-// oneOf / anyOf, or a "type" array expanded to one alternative per type
-struct common_chat_schema_any_of : common_chat_schema {
-    std::vector<common_chat_schema_ptr> children;
-
-    node_kind kind() const override { return KIND_ANY_OF; }
-};
-
-struct common_chat_schema_all_of : common_chat_schema {
-    std::vector<common_chat_schema_ptr> children;
-
-    node_kind kind() const override { return KIND_ALL_OF; }
-};
-
-struct common_chat_schema_const : common_chat_schema {
-    common_json value;
-
-    explicit common_chat_schema_const(common_json value) : value(std::move(value)) {}
-
-    node_kind kind() const override { return KIND_CONST; }
-};
-
-struct common_chat_schema_enum : common_chat_schema {
-    std::vector<common_json> values;
-
-    node_kind kind() const override { return KIND_ENUM; }
-};
-
-struct common_chat_schema_null : common_chat_schema {
-    node_kind kind() const override { return KIND_NULL; }
-};
-
-struct common_chat_schema_boolean : common_chat_schema {
-    node_kind kind() const override { return KIND_BOOLEAN; }
-};
-
-struct common_chat_schema_number : common_chat_schema {
-    node_kind kind() const override { return KIND_NUMBER; }
-};
-
-// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in
-struct common_chat_schema_integer : common_chat_schema {
-    int64_t minimum = INT64_MIN;  // INT64_MIN for unbounded
-    int64_t maximum = INT64_MAX;  // INT64_MAX for unbounded
-
-    node_kind kind() const override { return KIND_INTEGER; }
-};
-
-struct common_chat_schema_string : common_chat_schema {
-    std::string   pattern;  // empty when absent
-    string_format format     = FORMAT_NONE;
-    int           min_length = 0;
-    int           max_length = -1;  // -1 for unbounded
-
-    node_kind kind() const override { return KIND_STRING; }
-};
-
-struct common_chat_schema_array : common_chat_schema {
-    common_chat_schema_ptr items;  // a common_chat_schema_any when "items" is absent
-    int                    min_items = 0;
-    int                    max_items = -1;  // -1 for unbounded
-
-    node_kind kind() const override { return KIND_ARRAY; }
-};
-
-struct common_chat_schema_tuple : common_chat_schema {
-    std::vector<common_chat_schema_ptr> items;
-
-    node_kind kind() const override { return KIND_TUPLE; }
-};
-
-struct common_chat_schema_property {
-    std::string            name;
-    common_chat_schema_ptr schema;
-    bool                   required = false;
-};
-
-struct common_chat_schema_object : common_chat_schema {
-    std::vector<common_chat_schema_property> properties;             // in schema order
-    common_chat_schema_ptr                   additional_properties;  // null when not allowed
-
-    node_kind kind() const override { return KIND_OBJECT; }
-};
-
-struct common_chat_schema_document {
-    common_chat_schema_ptr                        root;
-    std::map<std::string, common_chat_schema_ptr> refs;
-};
-
-// A document shared by the PEG parsers built from its nodes, which it keeps alive
-using common_chat_schema_document_ptr = std::shared_ptr<const common_chat_schema_document>;
-
-// Throws std::runtime_error when the schema falls outside the supported subset.
-common_chat_schema_document common_chat_schema_from_json(const common_json & schema);

common/json.h

@@ -82,9 +82,6 @@ struct common_json_value {
     // note: a nested pair {"a", "b"} does not build, use common_json::array({"a", "b"}) for an array
     common_json_value(std::initializer_list<common_json_item> items);
 
-    template <typename T, typename std::enable_if<std::is_enum<T>::value, int>::type = 0>
-    common_json_value(T val) : common_json_value((typename std::underlying_type<T>::type) val) {}
-
     template <typename T, typename std::enable_if<std::is_integral<T>::value && !std::is_same<T, bool>::value, int>::type = 0>
     common_json_value(T val) : type(std::is_signed<T>::value ? VAL_INT : VAL_UINT) {
         if (std::is_signed<T>::value) {
@@ -114,7 +111,6 @@ struct common_json_item {
 // the types common_json_value holds on its own
 // anything else reaches its common_json ctor and recurses forever
 template <typename T> struct common_json_is_value : std::integral_constant<bool,
-    std::is_enum<T>::value ||
     std::is_arithmetic<T>::value ||
     std::is_same<T, std::nullptr_t>::value ||
     std::is_same<T, std::string>::value ||

common/log.cpp

@@ -1,6 +1,5 @@
 #include "common.h"
 #include "log.h"
-#include "json.h"
 
 #include <chrono>
 #include <condition_variable>
@@ -37,16 +36,6 @@ void common_log_set_verbosity_thold(int verbosity) {
     common_log_verbosity_thold = verbosity;
 }
 
-static bool common_log_jsonl = false;
-
-bool common_log_get_jsonl(void) {
-    return common_log_jsonl;
-}
-
-void common_log_set_jsonl(bool jsonl) {
-    common_log_jsonl = jsonl;
-}
-
 static int64_t t_us() {
     return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 }
@@ -77,17 +66,6 @@ static const char* g_col[] = {
     "",
 };
 
-static const char * level_str(enum ggml_log_level level) {
-    switch (level) {
-        case GGML_LOG_LEVEL_DEBUG: return "debug";
-        case GGML_LOG_LEVEL_INFO:  return "info";
-        case GGML_LOG_LEVEL_WARN:  return "warn";
-        case GGML_LOG_LEVEL_ERROR: return "error";
-        case GGML_LOG_LEVEL_CONT:  return "cont";
-        default:                   return "none";
-    }
-}
-
 struct common_log_entry {
     enum ggml_log_level level {GGML_LOG_LEVEL_INFO};
 
@@ -96,8 +74,6 @@ struct common_log_entry {
     int64_t timestamp { 0 };
     bool is_end       { false }; // signals the worker thread to stop
     bool prefix       { false };
-    bool jsonl        { false };
-    bool is_json      { false }; // msg already holds a serialized JSON object
 
     common_log_entry(size_t size = 256) : msg(size) { }
 
@@ -112,29 +88,11 @@ struct common_log_entry {
 
             fcur = stdout;
 
-            if (level != GGML_LOG_LEVEL_NONE && !jsonl) {
+            if (level != GGML_LOG_LEVEL_NONE) {
                 fcur = stderr;
             }
         }
 
-        if (jsonl) {
-            if (is_json) {
-                fprintf(fcur, "%s\n", msg.data());
-                fflush(fcur);
-                return;
-            }
-
-            common_json obj = {
-                {"type",  "log"},
-                {"time",  timestamp},
-                {"level", level_str(level)},
-                {"msg",   msg.data()},
-            };
-            fprintf(fcur, "%s\n", obj.dump_safe().c_str());
-            fflush(fcur);
-            return;
-        }
-
         if (level != GGML_LOG_LEVEL_NONE && level != GGML_LOG_LEVEL_CONT && prefix) {
             if (timestamp) {
                 // [M.s.ms.us]
@@ -288,8 +246,6 @@ public:
         entry.is_end    = false;
         entry.level     = level;
         entry.prefix    = prefix;
-        entry.jsonl     = common_log_jsonl;
-        entry.is_json   = false;
         entry.timestamp = 0;
         if (timestamps) {
             entry.timestamp = t_us() - t_start;
@@ -299,42 +255,6 @@ public:
         cv_new.notify_one();
     }
 
-    void add_json(const char * type, const common_json & obj) {
-        const common_json full = {
-            {"type", type},
-            {"data", obj},
-        };
-
-        const std::string text = full.dump_safe();
-
-        std::unique_lock<std::mutex> lock(mtx);
-
-        // block if the queue is full
-        cv_full.wait(lock, [this]() { return !running || !is_full(); });
-
-        if (!running) {
-            // discard messages while the worker thread is paused
-            return;
-        }
-
-        auto & entry = queue[tail];
-
-        if (entry.msg.size() < text.size() + 1) {
-            entry.msg.resize(text.size() + 1);
-        }
-        memcpy(entry.msg.data(), text.c_str(), text.size() + 1);
-
-        entry.is_end    = false;
-        entry.level     = GGML_LOG_LEVEL_NONE;
-        entry.prefix    = false;
-        entry.jsonl     = true;
-        entry.is_json   = true;
-        entry.timestamp = 0;
-
-        tail = (tail + 1) % queue.size();
-        cv_new.notify_one();
-    }
-
     void resume() {
         std::lock_guard<std::mutex> lock(mtx);
 
@@ -486,14 +406,6 @@ void common_log_add(struct common_log * log, enum ggml_log_level level, const ch
     va_end(args);
 }
 
-void common_log_add_json(struct common_log * log, const char * type, const common_json & obj) {
-    if (!common_log_jsonl) {
-        return;
-    }
-
-    log->add_json(type, obj);
-}
-
 void common_log_set_file(struct common_log * log, const char * file) {
     log->set_file(file);
 }
@@ -526,7 +438,7 @@ void common_log_flush(struct common_log * log) {
     log->resume();
 }
 
-int common_log_get_verbosity(enum ggml_log_level level) {
+static int common_get_verbosity(enum ggml_log_level level) {
     switch (level) {
         case GGML_LOG_LEVEL_DEBUG: return LOG_LEVEL_DEBUG;
         case GGML_LOG_LEVEL_INFO:  return LOG_LEVEL_TRACE;
@@ -540,7 +452,7 @@ int common_log_get_verbosity(enum ggml_log_level level) {
 }
 
 void common_log_default_callback(enum ggml_log_level level, const char * text, void * /*user_data*/) {
-    auto verbosity = common_log_get_verbosity(level);
+    auto verbosity = common_get_verbosity(level);
     if (verbosity <= common_log_verbosity_thold) {
         common_log_add(common_log_main(), level, "%s", text);
     }

common/log.h

@@ -43,12 +43,6 @@ int  common_log_get_verbosity_thold(void);
 
 void common_log_set_verbosity_thold(int verbosity); // not thread-safe
 
-bool common_log_get_jsonl(void);
-
-void common_log_set_jsonl(bool jsonl); // not thread-safe
-
-int common_log_get_verbosity(enum ggml_log_level level);
-
 void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data);
 
 // the common_log uses an internal worker thread to print/write log messages
@@ -130,17 +124,3 @@ void common_log_flush         (struct common_log * log);                    // f
 #define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  verbosity, __VA_ARGS__)
 #define LOG_ERRV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, verbosity, __VA_ARGS__)
 #define LOG_CNTV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_CONT,  verbosity, __VA_ARGS__)
-
-class common_json; // defined in common/json.h
-
-// helper allows different types of json output
-// no-op if --log-jsonl is not set
-void common_log_add_json(struct common_log * log, const char * type, const common_json & data);
-
-// will only print if --log-jsonl is set
-#define LOG_JSON(type, data) \
-    do { \
-        if (common_log_get_jsonl()) { \
-            common_log_add_json(common_log_main(), type, data); \
-        } \
-    } while (0)

common/parsers/cohere2moe.cpp

@@ -1,141 +0,0 @@
-#include "parsers.h"
-
-// Cohere2 MoE (a.k.a. "North Code") parser.
-//
-// The assistant turn is fully marker-wrapped:
-//   <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
-//     <|START_THINKING|>{reasoning}<|END_THINKING|>
-//     then EITHER content:    <|START_TEXT|>{content}<|END_TEXT|>
-//          OR     tool calls: <|START_ACTION|>[
-//                                 {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...
-//                             ]<|END_ACTION|>
-//   <|END_OF_TURN_TOKEN|>
-//
-// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is
-// the template default), so the model's output continues from *inside* the thinking block. The
-// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix
-// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,
-// regardless of whether they came from the generation prompt or the generated text.
-common_chat_params common_chat_params_init_cohere2moe(const common_chat_template &          tmpl,
-                                                              const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    const std::string TURN_START    = "<|START_OF_TURN_TOKEN|>";
-    const std::string TURN_END      = "<|END_OF_TURN_TOKEN|>";
-    const std::string CHATBOT       = "<|CHATBOT_TOKEN|>";
-    const std::string USER          = "<|USER_TOKEN|>";
-    const std::string SYSTEM        = "<|SYSTEM_TOKEN|>";
-    const std::string THINK_START   = "<|START_THINKING|>";
-    const std::string THINK_END     = "<|END_THINKING|>";
-    const std::string TEXT_START    = "<|START_TEXT|>";
-    const std::string TEXT_END      = "<|END_TEXT|>";
-    const std::string ACTION_START  = "<|START_ACTION|>";
-    const std::string ACTION_END    = "<|END_ACTION|>";
-    const std::string RESULT_START  = "<|START_TOOL_RESULT|>";
-    const std::string RESULT_END    = "<|END_TOOL_RESULT|>";
-
-    // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.
-    const std::string GEN_PREFIX = TURN_START + CHATBOT;
-
-    data.prompt             = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt  = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking  = true;
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = {THINK_END};
-    data.preserved_tokens   = {
-        TURN_START, TURN_END, CHATBOT, USER, SYSTEM,
-        THINK_START, THINK_END,
-        TEXT_START, TEXT_END,
-        ACTION_START, ACTION_END,
-        RESULT_START, RESULT_END,
-    };
-
-    // Declare per-role message delimiters. Tool results are rendered with the
-    // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before
-    // the plain "system" one (it is a strict superset, and the role split tries delimiters in order).
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX },
-        { COMMON_CHAT_ROLE_USER,      TURN_START + USER },
-        { COMMON_CHAT_ROLE_TOOL,      TURN_START + SYSTEM + RESULT_START },
-        { COMMON_CHAT_ROLE_SYSTEM,    TURN_START + SYSTEM },
-    };
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += THINK_END + TEXT_START + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal(GEN_PREFIX);
-        auto end               = p.end();
-
-        // The thinking block is always present (the generation prompt forces <|START_THINKING|>).
-        // When extracting reasoning, capture its body; otherwise keep the whole block (markers
-        // included) inline as content, matching reasoning_format=NONE conventions.
-        common_peg_parser reasoning = p.eps();
-        if (extract_reasoning) {
-            reasoning = p.optional(p.literal(THINK_START) +
-                                   p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) +
-                                   p.optional(p.literal(THINK_END)));
-        } else {
-            reasoning = p.optional(p.content(p.literal(THINK_START) +
-                                             p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) +
-                                             p.optional(p.literal(THINK_END))));
-        }
-
-        auto text_content = has_response_format
-            ? p.literal(TEXT_START) +
-                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
-                p.optional(p.literal(TEXT_END))
-            : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));
-
-        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-            return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;
-        }
-
-        auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-
-        // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>
-        auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,
-                                                /* force_tool_calls = */ true,
-                                                /* name_key         = */ "tool_name",
-                                                /* args_key         = */ "parameters",
-                                                /* array_wrapped    = */ true,
-                                                /* function_is_key  = */ false,
-                                                /* call_id_key      = */ "",
-                                                /* gen_call_id_key  = */ "tool_call_id",
-                                                /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });
-
-        // Content and tool calls are mutually exclusive in this format.
-        common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });
-
-        return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }
-        };
-    }
-
-    return data;
-}

common/parsers/deepseek.cpp

@@ -1,273 +0,0 @@
-#include "parsers.h"
-
-// The DeepSeek V4 reference implementation renders consecutive tool results into a single
-// user block, ordered by the tool call order of the preceding assistant message (matched
-// by tool call id) rather than by the order they appear in the conversation.
-static json deepseek_v4_sort_tool_results(const json & messages) {
-    json adjusted = messages;
-    std::map<std::string, size_t> call_order;
-
-    for (size_t i = 0; i < adjusted.size();) {
-        const auto & msg  = adjusted[i];
-        const auto   role = msg.value("role", "");
-
-        if (role == "assistant" && msg.contains("tool_calls") &&
-                msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
-            call_order.clear();
-            const auto & tool_calls = msg.at("tool_calls");
-            for (size_t idx = 0; idx < tool_calls.size(); idx++) {
-                auto id = tool_calls[idx].value("id", "");
-                if (!id.empty()) {
-                    call_order[id] = idx;
-                }
-            }
-            i++;
-            continue;
-        }
-
-        if (role != "user" && role != "tool") {
-            i++;
-            continue;
-        }
-
-        // collect a maximal run of user/tool messages - they render into one user block
-        std::vector<size_t> tool_positions;
-        size_t run_end = i;
-        for (; run_end < adjusted.size(); run_end++) {
-            const auto r = adjusted[run_end].value("role", "");
-            if (r == "tool") {
-                tool_positions.push_back(run_end);
-            } else if (r != "user") {
-                break;
-            }
-        }
-
-        if (tool_positions.size() > 1 && !call_order.empty()) {
-            std::vector<json> results;
-            results.reserve(tool_positions.size());
-            for (auto pos : tool_positions) {
-                results.push_back(adjusted[pos]);
-            }
-            std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {
-                const auto order = [&](const json & m) {
-                    auto it = call_order.find(m.value("tool_call_id", ""));
-                    return it == call_order.end() ? (size_t) 0 : it->second;
-                };
-                return order(a) < order(b);
-            });
-            for (size_t k = 0; k < tool_positions.size(); k++) {
-                adjusted[tool_positions[k]] = std::move(results[k]);
-            }
-        }
-
-        i = run_end;
-    }
-
-    return adjusted;
-}
-
-common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template &    tmpl,
-                                                                 const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"
-    // instead of "function_calls", renders tool results in tool call order and its
-    // non-thinking generation prompt ends with a bare </think> instead of an empty
-    // <think></think> pair.
-    const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
-
-    std::optional<json> adjusted_messages;
-    if (is_v4) {
-        adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
-    }
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    std::optional<json> additional_context;
-    if (is_v4 && has_response_format) {
-        additional_context = json{ { "response_format", inputs.json_schema } };
-    }
-
-    const std::string DSML         = "｜DSML｜";
-    const std::string THINK_START  = "<think>";
-    const std::string THINK_END    = "</think>";
-    const std::string TC_BLOCK     = is_v4 ? "tool_calls" : "function_calls";
-    const std::string FC_START     = "<" + DSML + TC_BLOCK + ">";
-    const std::string FC_END       = "</" + DSML + TC_BLOCK + ">";
-    const std::string INVOKE_START = "<" + DSML + "invoke";
-    const std::string INVOKE_END   = "</" + DSML + "invoke>";
-    const std::string PARAM_START  = "<" + DSML + "parameter";
-    const std::string PARAM_END    = "</" + DSML + "parameter>";
-    const std::string GEN_PROMPT   = "<｜Assistant｜>";
-    const std::string TC_SEPARATOR = "\n\n";
-
-    // lets the server find user turns in the prompt and place context checkpoints there
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, GEN_PROMPT },
-        { COMMON_CHAT_ROLE_USER, "<｜User｜>" },
-    };
-
-    data.prompt = common_chat_template_direct_apply_impl(
-        tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(
-        tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
-    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking  = true;
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = {THINK_END, FC_START};
-    data.preserved_tokens   = {
-        DSML,
-        THINK_START,
-        THINK_END,
-    };
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        if (is_v4 && msg.reasoning_content.empty()) {
-            data.generation_prompt = GEN_PROMPT + THINK_END;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += msg.render_content();
-            }
-        } else {
-            data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += THINK_END + msg.render_content();
-            }
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    bool require_tools   = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-    bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal(GEN_PROMPT);
-        auto end               = p.end();
-
-        // build tool call section first since we might need it in reasoning
-        auto tool_choice = p.choice();
-        if (has_tool_calls) {
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                std::string  name     = function.at("name");
-
-                std::vector<common_peg_parser> required_parsers;
-                std::vector<common_peg_parser> optional_parsers;
-                foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
-                    bool is_string = param.schema->may_be_string();
-
-                    auto arg = p.tool_arg(
-                        p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param.name)) +
-                                        p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
-                        (is_string ?
-                             p.tool_arg_string_value(p.until(PARAM_END)) :
-                             p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param.name + "-schema",
-                                                            doc, *param.schema))) +
-                        p.tool_arg_close(p.literal(PARAM_END)));
-
-                    auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
-                    if (param.required) {
-                        required_parsers.push_back(named_arg);
-                    } else {
-                        optional_parsers.push_back(named_arg);
-                    }
-                });
-
-                common_peg_parser args_seq = p.eps();
-                for (size_t i = 0; i < required_parsers.size(); i++) {
-                    if (i > 0) {
-                        args_seq = args_seq + p.space();
-                    }
-                    args_seq = args_seq + required_parsers[i];
-                }
-
-                if (!optional_parsers.empty()) {
-                    common_peg_parser any_opt = p.choice();
-                    for (const auto & opt : optional_parsers) {
-                        any_opt |= opt;
-                    }
-                    args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
-                }
-
-                common_peg_parser invoke_body = args_seq;
-                auto              func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") +
-                                                                   p.tool_name(p.literal(name)) + p.literal("\">\n")) +
-                                                       invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));
-
-                tool_choice |= p.rule("tool-" + name, func_parser);
-            });
-        }
-
-        common_peg_parser tool_calls = p.eps();
-        if (inputs.parallel_tool_calls) {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice +
-                p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
-        } else {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
-        }
-
-        auto reasoning = p.eps();
-        auto reasoning_with_tc = p.eps();
-        auto obligatory_tool_calls = tool_calls;
-        bool allow_reasoning_with_tc = false;
-
-        if (!require_tools) {
-            tool_calls = p.optional(tool_calls);
-        }
-
-        if (extract_reasoning && inputs.enable_thinking) {
-            reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
-            reasoning_with_tc = THINK_START +
-                p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
-                p.space() + obligatory_tool_calls;
-            allow_reasoning_with_tc = true;
-        } else if (extract_reasoning) {
-            // Thinking disabled but reasoning extraction requested: the generation prompt
-            // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
-            // must still be consumed.
-            reasoning = is_v4
-                ? p.optional(p.literal(THINK_END))
-                : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
-        }
-
-        if (has_response_format) {
-            auto response_format = p.rule("response-format",
-                p.literal("```json") + p.space() +
-                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
-                p.space() + p.literal("```"));
-            return generation_prompt + reasoning + response_format + end;
-        }
-
-        if (!has_tool_calls) {
-            return generation_prompt + reasoning + p.content(p.rest()) + end;
-        }
-
-        auto content_before_tools = p.negate(p.literal(THINK_START)) +
-            p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +
-            p.space();
-        return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :
-            generation_prompt + reasoning + content_before_tools + tool_calls + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = has_tools && !require_tools;
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
-        };
-    }
-
-    return data;
-}