test: move tools/parser to tests

.devops/nix/package.nix

@@ -31,7 +31,7 @@
     ]
     && blas.meta.available,
   useCuda ? config.cudaSupport,
-  useMetalKit ? stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isDarwin,
+  useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin,
   # Increases the runtime closure size by ~700M
   useMpi ? false,
   useRocm ? config.rocmSupport,
@@ -92,7 +92,7 @@ let
 
   cudaBuildInputs = with cudaPackages; [
     cuda_cudart
-    cccl # <nv/target>
+    cuda_cccl # <nv/target>
     libcublas
   ];
 
@@ -166,7 +166,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
   # `xcrun` is used find the path of the Metal compiler, which is varible
   # and not on $PATH
   # see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
-  __noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
+  __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
 
   nativeBuildInputs =
     [
@@ -181,10 +181,10 @@ effectiveStdenv.mkDerivation (finalAttrs: {
       autoAddDriverRunpath
     ]
     ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
-    ++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
+    ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
 
   buildInputs =
-    optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
+    optionals effectiveStdenv.isDarwin darwinBuildInputs
     ++ optionals useCuda cudaBuildInputs
     ++ optionals useMpi [ mpi ]
     ++ optionals useRocm rocmBuildInputs
@@ -245,7 +245,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
 
     # Configurations that are known to result in build failures. Can be
     # overridden by importing Nixpkgs with `allowBroken = true`.
-    broken = (useMetalKit && !effectiveStdenv.hostPlatform.isDarwin);
+    broken = (useMetalKit && !effectiveStdenv.isDarwin);
 
     description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
     homepage = "https://github.com/ggml-org/llama.cpp/";

.devops/openvino.Dockerfile

@@ -1,12 +1,12 @@
-ARG OPENVINO_VERSION_MAJOR=2026.3.1
-ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
+ARG OPENVINO_VERSION_MAJOR=2026.3
+ARG OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c
 ARG UBUNTU_VERSION=24.04
 
 # Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
-ARG IGC_VERSION=v2.40.13
-ARG IGC_VERSION_FULL=2_2.40.13+22418
-ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
-ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
+ARG IGC_VERSION=v2.38.2
+ARG IGC_VERSION_FULL=2_2.38.2+22051
+ARG COMPUTE_RUNTIME_VERSION=26.27.39122.11
+ARG COMPUTE_RUNTIME_VERSION_FULL=26.27.39122.11-0
 ARG IGDGMM_VERSION=22.10.0
 
 # Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
@@ -90,9 +90,6 @@ RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
     cmake -B build/ReleaseOV -G Ninja \
         -DCMAKE_BUILD_TYPE=Release \
         -DLLAMA_BUILD_TESTS=OFF \
-        -DGGML_NATIVE=OFF \
-        -DGGML_BACKEND_DL=ON \
-        -DGGML_CPU_ALL_VARIANTS=ON \
         -DGGML_OPENVINO=ON && \
     cmake --build build/ReleaseOV --parallel "
 

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
-blank_issues_enabled: false
+blank_issues_enabled: true
 contact_links:
   - name: Got an idea?
     url: https://github.com/ggml-org/llama.cpp/discussions/categories/ideas

.github/actions/ccache-buckets/action.yml

@@ -1,95 +0,0 @@
-name: "ccache-buckets"
-description: "Save/restore latest GitHub Actions ccache matching a key prefix to/from HF buckets"
-inputs:
-  key:
-    description: "Cache key prefix to match and load"
-    required: true
-  folder:
-    description: "Bucket folder containing ccache files"
-    required: true
-  evict-old-files:
-    description: "Corresponds to the ccache --evict-older-than AGE option, where AGE is the number of seconds or days followed by the 's' or 'd' suffix respectively."
-    default: ''
-  save:
-    description: "Save ccache"
-    required: false
-    default: false
-    type: boolean
-  hf_bucket:
-    description: 'Hugging Face buckets path'
-    required: true
-
-runs:
-  using: "composite"
-  steps:
-    - name: Install Hugging Face Hub CLI
-      shell: bash
-      run: |
-        python3 -m venv .venv-hf
-        .venv-hf/bin/pip install -U huggingface_hub==1.28.0
-
-    - name: Restore ccache from buckets
-      if: ${{ inputs.save != 'true' }}
-      shell: bash
-      run: |
-        set +e -uo pipefail
-        source .venv-hf/bin/activate
-        CCACHE_DIR=$(ccache -k cache_dir)
-        if [[ -d "$CCACHE_DIR" ]]; then
-            CACHE_PATH=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path) | last | .path // ""')
-            if [[ -n "$CACHE_PATH" ]]; then
-                echo "Restoring ccache from '$CACHE_PATH'."
-                hf buckets cp "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" ccache_bucket.tar.gz
-                mkdir -p ccache_bucket
-                if tar -xzf ccache_bucket.tar.gz -C ccache_bucket; then
-                    rm -rf "$CCACHE_DIR"
-                    mv ccache_bucket "$CCACHE_DIR"
-                    ccache -z
-                fi
-                rm ccache_bucket.tar.gz
-            else
-                echo "No ccache found."
-            fi
-        else
-            echo "'$CCACHE_DIR' not found."
-        fi
-
-    - name: Save ccache to buckets
-      if: ${{ inputs.save == 'true' }}
-      shell: bash
-      run: |
-        if [[ -n "$HF_TOKEN" ]]; then
-            set +e -uo pipefail
-            source .venv-hf/bin/activate
-            CCACHE_DIR=$(ccache -k cache_dir)
-            if [[ -d "$CCACHE_DIR" ]]; then
-                ccache -s
-                if [[ -n "${{ inputs.evict-old-files }}" ]]; then
-                    ccache --evict-older-than "${{ inputs.evict-old-files }}"
-                fi
-                DATESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-                CACHEFILE="${{ inputs.key }}-$DATESTAMP.tar.gz"
-                if tar -czf ccache_bucket.tar.gz -C "$CCACHE_DIR" .; then
-                    hf buckets cp ccache_bucket.tar.gz "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}/$CACHEFILE"
-                fi
-                rm ccache_bucket.tar.gz
-            else
-                echo "'$CCACHE_DIR' not found."
-            fi
-        fi
-
-    - name: Remove old ccache files from buckets
-      if: ${{ inputs.save == 'true' }}
-      shell: bash
-      run: |
-        if [[ -n "$HF_TOKEN" ]]; then
-            set +e -uo pipefail
-            source .venv-hf/bin/activate
-            CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select((.uploaded_at | .[:19]+"Z" | fromdateiso8601) < (now - 5 * 60)) | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv')
-            if [[ -n "$CACHE_FILES" ]]; then
-                echo "Removing old ccache files..."
-                while IFS=$'\t' read -r CACHE_PATH; do
-                    hf buckets rm "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" -y
-                done <<< "$CACHE_FILES"
-            fi
-        fi

.github/actions/ccache-clear/action.yml

@@ -21,30 +21,68 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install GitHub CLI if missing
-      shell: bash
-      run: |
-        # e.g. in container jobs, where it is not preinstalled
-        if ! command -v gh >/dev/null 2>&1; then
-          echo "GitHub CLI not found, installing..."
-          if ! command -v curl >/dev/null 2>&1; then
-            apt-get update >/dev/null 2>&1 || true
-            apt-get install -y curl >/dev/null 2>&1 || true
-          fi
-          mkdir -p -m 755 /etc/apt/keyrings
-          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg >/dev/null
-          chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list
-          apt-get update >/dev/null 2>&1 || true
-          apt-get install -y gh || { echo "Failed to install GitHub CLI (gh)" >&2; exit 1; }
-        fi
-        command -v gh >/dev/null 2>&1 || { echo "GitHub CLI (gh) is required but could not be installed" >&2; exit 1; }
-
     - name: Clear caches
       shell: bash
+      env:
+        CLEAR_KEY: ${{ inputs.key }}
+        CLEAR_OLDER: ${{ inputs.older }}
+        CLEAR_MIN: ${{ inputs.min }}
+        CLEAR_DRY_RUN: ${{ inputs.dry-run }}
       run: |
-        bash scripts/ccache-clear.sh \
-          --key "${{ inputs.key }}" \
-          --older "${{ inputs.older }}" \
-          --min "${{ inputs.min }}" \
-          ${{ inputs.dry-run == 'true' && '--dry-run' || '' }}
+        # Convert a duration (e.g. 90m, 1h, 1d, plain seconds) to seconds
+        to_seconds() {
+          local val="$1"
+          [[ "$val" =~ ^[0-9]+$ ]] && { echo "$val"; return 0; }
+          local num="${val%?}" unit="${val: -1}" mult
+          [[ "$num" =~ ^[0-9]+$ ]] || return 1
+          case "$unit" in
+            s) mult=1 ;;
+            m) mult=60 ;;
+            h) mult=3600 ;;
+            d) mult=86400 ;;
+            *) return 1 ;;
+          esac
+          echo $((num * mult))
+        }
+
+        [[ "$CLEAR_MIN" =~ ^[0-9]+$ ]] || { echo "Invalid min value: $CLEAR_MIN" >&2; exit 1; }
+        [[ "$CLEAR_DRY_RUN" =~ ^(true|false)$ ]] || { echo "Invalid dry-run value: $CLEAR_DRY_RUN" >&2; exit 1; }
+
+        CACHES=$(gh cache list --key "ccache-$CLEAR_KEY" --json id,key,createdAt --jq '.[] | [.createdAt, .id, .key] | @tsv' 2>/dev/null | LC_ALL=C sort)
+        if [ -z "$CACHES" ]; then
+          echo "No caches found with key prefix: $CLEAR_KEY"
+          exit 0
+        fi
+
+        TOTAL=$(( $(wc -l <<< "$CACHES") ))
+
+        echo "Found $TOTAL cache(s) with key prefix: $CLEAR_KEY (oldest first):"
+        while IFS=$'\t' read -r CREATED ID KEY; do
+          printf '  %s  %s  %s\n' "$CREATED" "$ID" "$KEY"
+        done <<< "$CACHES"
+
+        CUTOFF=""
+        if [ -n "$CLEAR_OLDER" ]; then
+          OLDER_SECONDS=$(to_seconds "$CLEAR_OLDER") || { echo "Invalid older value: $CLEAR_OLDER (expected e.g. 90m, 1h, 1d)" >&2; exit 1; }
+          CUTOFF=$(( $(date +%s) - OLDER_SECONDS ))
+        fi
+
+        # Caches are sorted oldest first
+        DELETED=0
+        while IFS=$'\t' read -r CREATED ID KEY; do
+          if [ -n "$CUTOFF" ] && [ "$(date -d "$CREATED" +%s)" -ge "$CUTOFF" ]; then
+            echo "Rest are not older than $CLEAR_OLDER, stopping"
+            break
+          fi
+          if [ $((TOTAL - DELETED - 1)) -lt "$CLEAR_MIN" ]; then
+            echo "Keeping at least $CLEAR_MIN cache(s), stopping"
+            break
+          fi
+          if [ "$CLEAR_DRY_RUN" = "true" ]; then
+            echo "Would delete cache: $ID ($KEY)"
+          else
+            echo "Deleting cache: $ID ($KEY)"
+            gh cache delete "$ID"
+          fi
+          DELETED=$((DELETED + 1))
+        done <<< "$CACHES"

.github/actions/windows-setup-cuda/action.yml

@@ -137,19 +137,19 @@ runs:
       run: |
           mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
           choco install unzip -y
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cccl-windows-x86_64-13.3.4.1.2-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_crt-windows-x86_64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_nvcc-windows-x86_64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-x86_64/5B515474-7E78-11F1-8656-C51E4F4B317F/libnvvm-windows-x86_64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/cuda_cudart-windows-arm64-13.4.46-archive.zip"
+          curl -O "https://packages.nvidia.com/bin-archive/pool/windows-arm64/5B515474-7E78-11F1-8656-C51E4F4B317F/libcublas-windows-arm64-13.7.0.10-archive.zip"
           unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.1.2-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.46-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.10-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
           echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
           echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
           echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8

.github/actions/windows-setup-rocm/action.yml

@@ -24,7 +24,7 @@ runs:
 
         write-host "Installing ROCm wheels for multi-arch support"
         # Install ROCm wheels for multi-arch support (this may take several minutes)
-        python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ inputs.version }}"
+        python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ inputs.version }}"
 
         # Pre-expand the devel tree so it is included in the cache
         write-host "Initializing ROCm devel tree"

.github/workflows/build-android.yml

@@ -110,7 +110,7 @@ jobs:
       #   cache on:  https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
       #
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.21
       #  with:
       #    key: android-ubuntu-arm64
       #    evict-old-files: 1d

.github/workflows/build-apple.yml

@@ -22,8 +22,7 @@ on:
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-apple.yml',
-      'ggml/src/ggml-metal/**',
-      'ggml/src/ggml-rpc/**'
+      'ggml/src/ggml-metal/**'
     ]
 
 concurrency:
@@ -47,19 +46,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: apple-arm64
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: apple-arm64
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -74,25 +65,7 @@ jobs:
             -DGGML_RPC=ON \
             -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
           time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: apple-arm64
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      - name: Check for leaks
-        run: |
-          cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1)
-          leaks -atExit -- "${cmd[@]}"
-          # Graphics devices are leaked by Metal in Apple code sometimes, so we ignore those leaks
-          OBJC_DEBUG_MISSING_POOLS=YES "${cmd[@]}" 2>&1 | awk '{ print } index($0, "autoreleased with no pool in place") && !/class [a-zA-Z0-9]+Device autoreleased/ { found = 1 } END { exit found }'
+          leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1
 
       - name: Test
         id: cmake_test
@@ -109,19 +82,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: apple-x64
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: apple-x64
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -138,18 +103,6 @@ jobs:
             -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
           time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: apple-x64
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
@@ -210,6 +163,14 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
+      # TODO: this likely does not do anything - if yes, remove it
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          key: apple-tvos
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
       - name: Build
         id: cmake_build
         run: |
@@ -235,6 +196,14 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
+      # TODO: this likely does not do anything - if yes, remove it
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          key: apple-visionos
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
       - name: Build
         id: cmake_build
         run: |
@@ -265,6 +234,14 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
+      # TODO: this likely does not do anything - if yes, remove it
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          key: apple-swift
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
       - name: Download xcframework artifact
         uses: actions/download-artifact@v7
         with:

.github/workflows/build-cache.yml

@@ -41,8 +41,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.3.1"
-      OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone
@@ -69,8 +69,8 @@ jobs:
 
     env:
       # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.3.1"
-      OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone

.github/workflows/build-cpu.yml

@@ -62,10 +62,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cpu-${{ matrix.os }}
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build Dependencies
         id: build_depends
@@ -90,15 +91,6 @@ jobs:
           python3 -m pip install --upgrade pip setuptools
           pip3 install ./gguf-py
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: cpu-${{ matrix.os }}
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
@@ -108,18 +100,6 @@ jobs:
             -DGGML_RPC=ON
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: cpu-${{ matrix.os }}
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
@@ -137,6 +117,18 @@ jobs:
           ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
           ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
 
+      # note: real deletion only on push to master (same condition as the ccache save),
+      #       dry-run otherwise (the token is read-only on PRs from forks)
+      - name: ccache-clear
+        uses: ./.github/actions/ccache-clear
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          key: cpu-${{ matrix.os }}
+          older: 1h
+          min: 1
+          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+
   windows:
     name: windows / ${{ matrix.build }}
     runs-on: windows-2025
@@ -164,7 +156,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cpu-windows-2025-${{ matrix.build }}
           variant: ccache
@@ -221,14 +213,5 @@ jobs:
       #     7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
       #     $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
       #     cd build
+      #     $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
       #     & $sde -future -- ctest -L main -C Release --verbose --timeout 900
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: cpu-windows-2025-${{ matrix.build }}
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/build-cuda-ubuntu.yml

@@ -50,22 +50,14 @@ jobs:
           DEBIAN_FRONTEND: noninteractive
         run: |
           apt update
-          apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip
+          apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-24.04-cuda
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: cuda-ubuntu-24.04-cuda
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build with CMake
         # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
@@ -80,18 +72,6 @@ jobs:
             -DGGML_CUDA_CUB_3DOT2=ON
           cmake --build build
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: cuda-ubuntu-24.04-cuda
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
   hip:
     runs-on: ubuntu-22.04
     container: rocm/dev-ubuntu-22.04:6.1.2
@@ -105,22 +85,14 @@ jobs:
         id: depends
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv
+          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-22.04-hip
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: cuda-ubuntu-22.04-hip
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build with native CMake HIP support
         id: cmake_build
@@ -131,18 +103,6 @@ jobs:
             -DGGML_HIP=ON
           cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: cuda-ubuntu-22.04-hip
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
   musa:
     runs-on: ubuntu-22.04
     container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
@@ -156,22 +116,14 @@ jobs:
         id: depends
         run: |
           apt-get update
-          apt-get install -y build-essential git cmake libssl-dev jq
+          apt-get install -y build-essential git cmake libssl-dev
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cuda-ubuntu-22.04-musa
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: cuda-ubuntu-22.04-musa
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build with native CMake MUSA support
         id: cmake_build
@@ -179,15 +131,3 @@ jobs:
           cmake -B build -S . \
             -DGGML_MUSA=ON
           time cmake --build build --config Release -j $(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: cuda-ubuntu-22.04-musa
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true

.github/workflows/build-cuda-windows.yml

@@ -47,7 +47,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
 
@@ -152,7 +152,7 @@ jobs:
           & "${env:HIP_PATH}\lib\llvm\bin\clang.exe" --version
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           # TODO: this build does not match the build in release.yml, so we use a different cache key
           #       ideally, the builds should match, similar to the CUDA build above so that we would be able

.github/workflows/build-msys.yml

@@ -35,7 +35,7 @@ jobs:
         uses: actions/checkout@v6
 
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.16
       #  with:
       #    key: msys-windows-2025-x64
       #    variant: ccache

.github/workflows/build-opencl.yml

@@ -44,7 +44,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: opencl-windows-2025-x64
           variant: ccache
@@ -80,13 +80,3 @@ jobs:
         run: |
           cmake -S . -B build -G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON -DLLAMA_BUILD_BORINGSSL=ON
           cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: opencl-windows-2025-x64
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/build-openvino.yml

@@ -32,8 +32,6 @@ env:
   LLAMA_ARG_LOG_COLORS: 1
   LLAMA_ARG_LOG_PREFIX: 1
   LLAMA_ARG_LOG_TIMESTAMPS: 1
-  # TODO: fix failing tests on OpenVINO backend
-  CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-save-load-state"
 
 jobs:
   ubuntu-24-openvino:
@@ -41,8 +39,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.3.1"
-      OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone
@@ -80,24 +78,26 @@ jobs:
 
       - name: Test (CPU)
         id: cmake_test_cpu
+        # TODO: fix and re-enable the `test-llama-archs` test below
         run: |
           cd ${{ github.workspace }}
-          ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
+          ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 2000
 
       - name: Test (GPU)
         id: cmake_test_gpu
+        # TODO: fix and re-enable the `test-llama-archs` test below
         run: |
           cd ${{ github.workspace }}
           export GGML_OPENVINO_DEVICE=GPU
-          ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
+          ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 3000
 
   openvino-windows-2022:
     runs-on: windows-2022
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.3.1"
-      OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone
@@ -105,7 +105,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: openvino-windows-2022
           variant: ccache
@@ -159,20 +159,11 @@ jobs:
       - name: Test (CPU)
         id: cmake_test_cpu
         shell: cmd
+        # TODO: fix and re-enable the `test-llama-archs` test below
         run: |
           REM Find extracted OpenVINO folder dynamically
           for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
           call "%OPENVINO_ROOT%\setupvars.bat"
 
           cd build
-          ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: openvino-windows-2022
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
+          ctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000

.github/workflows/build-riscv.yml

@@ -67,7 +67,7 @@ jobs:
 
       # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
       #  with:
       #    key: riscv-ubuntu-native
       #    evict-old-files: 1d
@@ -106,3 +106,83 @@ jobs:
           wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
           ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
           ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
+
+  ubuntu-riscv64-native-sanitizer:
+    runs-on: ubuntu-24.04-riscv
+
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        sanitizer: [ADDRESS, THREAD, UNDEFINED]
+        build_type: [Debug]
+
+    steps:
+      - name: Install dependencies
+        run: |
+          # Set gcc-14 and g++-14 as the default compilers
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
+
+          git lfs install
+
+      - name: GCC version check
+        run: |
+          gcc --version
+          g++ --version
+
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
+      #- name: ccache
+      #  uses: ggml-org/ccache-action@afde29e5b5422e5da23cb1f639e8baecadeadfc3 # https://github.com/ggml-org/ccache-action/pull/1
+      #  with:
+      #    key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }}
+      #    evict-old-files: 1d
+      #    save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+
+      - name: Build
+        id: cmake_build
+        if: ${{ matrix.sanitizer != 'THREAD' }}
+        run: |
+          cmake -B build \
+            -DLLAMA_OPENSSL=OFF \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DGGML_OPENMP=ON \
+            -DLLAMA_BUILD_EXAMPLES=ON \
+            -DLLAMA_BUILD_TOOLS=ON \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
+
+          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+
+      - name: Build (no OpenMP)
+        id: cmake_build_no_openmp
+        if: ${{ matrix.sanitizer == 'THREAD' }}
+        run: |
+          cmake -B build \
+            -DLLAMA_OPENSSL=OFF \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DGGML_OPENMP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=ON \
+            -DLLAMA_BUILD_TOOLS=ON \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
+            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
+
+          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+
+      - name: Test
+        id: cmake_test
+        run: |
+          cd build
+          ctest -L main --verbose --timeout 900

.github/workflows/build-sanitize.yml

@@ -55,7 +55,7 @@ jobs:
         uses: actions/checkout@v6
 
       # - name: ccache
-      #   uses: ggml-org/ccache-action@v1.2.24
+      #   uses: ggml-org/ccache-action@v1.2.21
       #   if: ${{ matrix.sanitizer != 'UNDEFINED' }}
       #   with:
       #     key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
@@ -101,6 +101,8 @@ jobs:
 
       - name: Test
         id: cmake_test
+        # skip run in Debug - very slow
+        if: ${{ matrix.sanitizer != 'UNDEFINED' }}
         run: |
           cd build
           ctest -L main -E tokenizer --verbose --timeout 900

.github/workflows/build-self-hosted.yml

@@ -58,48 +58,18 @@ env:
 
 jobs:
   gpu-cuda:
-    runs-on: "hf-jobs-t4-small:cuda13"
+    runs-on: [self-hosted, Linux, NVIDIA]
 
     steps:
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
 
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y cmake libssl-dev time unzip wget python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: self-hosted-gpu-cuda
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Test
         id: ggml-ci
         run: |
           nvidia-smi
-          GG_BUILD_CUDA=1 CUDACXX=/usr/local/cuda/bin/nvcc bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: self-hosted-gpu-cuda
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+          GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
 
   gpu-rocm:
     runs-on: [self-hosted, Linux, AMD]
@@ -160,10 +130,10 @@ jobs:
       - name: Dawn Dependency
         id: dawn-depends
         run: |
-          DAWN_VERSION="v20260908.214631"
+          DAWN_VERSION="v20260317.182325"
           DAWN_OWNER="google"
           DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
           echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           curl -L -o artifact.tar.gz \
             "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
@@ -246,10 +216,10 @@ jobs:
       - name: Dawn Dependency
         id: dawn-depends
         run: |
-          DAWN_VERSION="v20260908.214631"
+          DAWN_VERSION="v20260317.182325"
           DAWN_OWNER="google"
           DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
           echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           curl -L -o artifact.tar.gz \
             "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
@@ -318,8 +288,8 @@ jobs:
 
     env:
       # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.3.1"
-      OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Clone
@@ -395,11 +365,7 @@ jobs:
       - name: Test
         id: ggml-ci
         run: |
-          LLAMA_ARG_THREADS=$(nproc) \
-          GG_BUILD_HIGH_PERF=1 \
-          GG_BUILD_NO_BF16=1 \
-          GG_BUILD_EXTRA_TESTS_0=1 \
-          bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
 
   cpu-arm64-graviton4-kleidiai:
     runs-on: ah-ubuntu_22_04-c8g_8x
@@ -438,8 +404,6 @@ jobs:
       - name: Test
         id: ggml-ci
         run: |
-          LLAMA_ARG_THREADS=$(nproc) \
           GG_BUILD_KLEIDIAI=1 \
           GG_BUILD_EXTRA_TESTS_0=1 \
-          GG_BUILD_HIGH_PERF=1 \
-          bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
+          bash ./ci/run.sh ./tmp/results ./tmp/mnt

.github/workflows/build-sycl.yml

@@ -75,19 +75,11 @@ jobs:
           sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: sycl-ubuntu-24-${{ matrix.build }}
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: sycl-ubuntu-24-${{ matrix.build }}
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -104,18 +96,6 @@ jobs:
             -DGGML_SYCL_F16=${{ matrix.fp16 }}
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: sycl-ubuntu-24-${{ matrix.build }}
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
   windows-latest-sycl:
     runs-on: windows-2022
 
@@ -147,7 +127,7 @@ jobs:
           "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: sycl-windows-latest
           variant: ccache
@@ -159,13 +139,3 @@ jobs:
       - name: Build
         id: cmake_build
         run:  examples/sycl/win-build-sycl.bat
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: sycl-windows-latest
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/build-vulkan.yml

@@ -53,20 +53,12 @@ jobs:
           echo "CXX=g++-14" >> "$GITHUB_ENV"
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
-          key: vulkan-ubuntu-24.04-arm
+          key: vulkan-ubuntu-24.04-arm-new
           variant: ccache
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: vulkan-ubuntu-24.04-arm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Configure
         id: cmake_configure
@@ -81,18 +73,6 @@ jobs:
         run: |
           time cmake --build build -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: vulkan-ubuntu-24.04-arm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
   ubuntu-llvmpipe:
     runs-on: ubuntu-24.04
 
@@ -122,19 +102,11 @@ jobs:
           strip: 1
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: vulkan-ubuntu-24.04-llvmpipe
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: vulkan-ubuntu-24.04-llvmpipe
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -145,18 +117,6 @@ jobs:
             -DGGML_VULKAN=ON
           cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: vulkan-ubuntu-24.04-llvmpipe
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
@@ -164,7 +124,9 @@ jobs:
           export GGML_VK_VISIBLE_DEVICES=0
           export GGML_VK_DISABLE_F16=1
           export GGML_VK_DISABLE_COOPMAT=1
-          ctest -L main --verbose --timeout 900
+          # This is using llvmpipe and runs slower than other backends
+          # test-backend-ops is too slow on llvmpipe, skip it
+          ctest -L main -E test-backend-ops --verbose --timeout 900
 
   windows:
     runs-on: windows-2025
@@ -178,7 +140,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: cpu-windows-2025-x64-vulkan
           variant: ccache
@@ -218,13 +180,3 @@ jobs:
         run: |
           cd build
           ctest -L main -C Release --verbose --timeout 900
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: cpu-windows-2025-x64-vulkan
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/build-wasm.yml

@@ -54,10 +54,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: webgpu-ubuntu-24.04-arm-wasm
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Install Emscripten
         run: |
@@ -68,22 +69,13 @@ jobs:
 
       - name: Fetch emdawnwebgpu
         run: |
-          DAWN_TAG="v20260908.214631"
+          DAWN_TAG="v20260317.182325"
           EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip"
           echo "Downloading ${EMDAWN_PKG}"
           curl -L -o emdawn.zip \
             "https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"
           unzip emdawn.zip
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: webgpu-ubuntu-24.04-arm-wasm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build WASM WebGPU
         run: |
           source emsdk/emsdk_env.sh
@@ -96,15 +88,3 @@ jobs:
             -DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg
 
           time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: webgpu-ubuntu-24.04-arm-wasm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true

.github/workflows/build-webgpu.yml

@@ -69,33 +69,25 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: webgpu-macos-latest
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Dawn Dependency
         id: dawn-depends
         run: |
-          DAWN_VERSION="v20260908.214631"
+          DAWN_VERSION="v20260317.182325"
           DAWN_OWNER="google"
           DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-macos-latest-Release"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
           echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           curl -L -o artifact.tar.gz \
             "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           mkdir dawn
           tar -xvf artifact.tar.gz -C dawn --strip-components=1
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: webgpu-macos-latest
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
@@ -103,18 +95,6 @@ jobs:
           cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
           time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: webgpu-macos-latest
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
@@ -130,10 +110,11 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: webgpu-ubuntu-24.04
-          save: false
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Dependencies
         id: depends
@@ -147,25 +128,16 @@ jobs:
         id: dawn-depends
         run: |
           sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
-          DAWN_VERSION="v20260908.214631"
+          DAWN_VERSION="v20260317.182325"
           DAWN_OWNER="google"
           DAWN_REPO="dawn"
-          DAWN_ASSET_NAME="Dawn-94c3c9cc0d5fb2e85aebb370fa8d37b71aa34655-ubuntu-latest-Release"
+          DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
           echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           curl -L -o artifact.tar.gz \
             "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
           mkdir dawn
           tar -xvf artifact.tar.gz -C dawn --strip-components=1
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: webgpu-ubuntu-24.04
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
@@ -174,20 +146,10 @@ jobs:
             -DGGML_WEBGPU=ON
           time cmake --build build --config Release -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: webgpu-ubuntu-24.04
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |
           cd build
-          ctest -L main --verbose --timeout 900
+          # This is using llvmpipe and runs slower than other backends
+          # test-backend-ops is too slow on llvmpipe, skip it
+          ctest -L main -E test-backend-ops --verbose --timeout 900

.github/workflows/copilot-setup-steps.yml

@@ -29,7 +29,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: copilot-setup-steps
           evict-old-files: 1d

.github/workflows/docker.yml

@@ -64,7 +64,7 @@ jobs:
     needs: create_tag
     uses: ./.github/workflows/ui-build.yml
     with:
-      ui_version: ${{ needs.create_tag.outputs.source_tag }}
+      hf_ui_version: ${{ needs.create_tag.outputs.source_tag }}
 
   prepare_matrices:
     name: Prepare Docker matrices
@@ -162,7 +162,7 @@ jobs:
         if: ${{ matrix.config.prebuilt_ui == true }}
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist
 
       - name: Set up QEMU

.github/workflows/fusion.yml

@@ -1,67 +0,0 @@
-name: Fusion
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/fusion.yml',
-      'ggml/**',
-      'tests/fusion/**',
-      'tests/test-fusion.cpp'
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/fusion.yml',
-      'ggml/**',
-      'tests/fusion/**',
-      'tests/test-fusion.cpp'
-    ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  # TODO: add jobs for other backends as they adopt the fusion debug API
-  metal:
-    runs-on: [self-hosted, macOS, ARM64]
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DGGML_SCHED_NO_REALLOC=ON \
-            -DGGML_BLAS=OFF \
-            -DGGML_METAL=ON
-          time cmake --build build --config Release --target test-llama-archs -j $(sysctl -n hw.logicalcpu)
-          time cmake --build build --config Release --target test-fusion -j $(sysctl -n hw.logicalcpu)
-
-      - name: Generate models
-        id: generate_models
-        run: |
-          rm -rf build-ci-models && mkdir -p build-ci-models
-          ./build/bin/test-llama-archs -o build-ci-models
-
-      - name: Test fusion
-        id: test_fusion
-        run: |
-          ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv

.github/workflows/hip-quality-check.yml

@@ -49,22 +49,14 @@ jobs:
         id: depends
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3 python3-venv python3-pip jq
+          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev python3
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: hip-quality-check-ubuntu-22.04
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: hip-quality-check-ubuntu-22.04
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build with Werror
         id: cmake_build
@@ -92,15 +84,3 @@ jobs:
           cd build
           make -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$'
           python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: hip-quality-check-ubuntu-22.04
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true

.github/workflows/make-release.yml

@@ -19,7 +19,6 @@ env:
 
 permissions:
   contents: write
-  packages: write
 
 jobs:
   make-release:
@@ -85,13 +84,11 @@ jobs:
 
             New version has been released.
 
-            ## Assets
-
             ${{ steps.desc.outputs.nightly }}
 
-            ## More info
+            **Web UI:** the `nightly-tag.txt` asset contains the tag of the corresponding nightly release
 
-            - [Releases and versioning of `ggml-org` projects](https://github.com/ggml-org/ggml/discussions/1579)
+            **More info:** [dist : releases and versioning of ggml-org projects](https://github.com/ggml-org/ggml/discussions/1579)
 
             ## ${{ steps.desc.outputs.changelog_title }}
 
@@ -114,29 +111,6 @@ jobs:
               data: await fs.readFileSync('./nightly-tag.txt')
             });
 
-      - name: Re-tag container images with release version
-        if: ${{ github.event.inputs.dry_run == 'false' && steps.desc.outputs.nightly_tag != '' }}
-        env:
-          GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
-        run: |
-          VERSION="${{ steps.checks.outputs.version }}"
-          NIGHTLY_TAG="${{ steps.desc.outputs.nightly_tag }}"
-          REPO_OWNER="${GITHUB_REPOSITORY_OWNER,,}"
-          IMAGE_REPO="ghcr.io/${REPO_OWNER}/${{ github.event.repository.name }}"
-
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
-
-          VARIANTS=("" "-cuda" "-cuda13" "-vulkan" "-rocm" "-intel" "-musa" "-openvino")
-          TYPES=("full" "light" "server")
-          for type in "${TYPES[@]}"; do
-            for variant in "${VARIANTS[@]}"; do
-              src="${IMAGE_REPO}:${type}${variant}-${NIGHTLY_TAG}"
-              dst="${IMAGE_REPO}:${type}${variant}-${VERSION}"
-              echo "Tagging ${src} -> ${dst}"
-              docker buildx imagetools create --tag "${dst}" "${src}"
-            done
-          done
-
       - name: Dry run summary
         if: ${{ github.event.inputs.dry_run == 'true' }}
         run: |

.github/workflows/python-type-check.yml

@@ -31,7 +31,7 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.11"
-          pip-install: -r requirements/requirements-all.txt ty==0.0.78
+          pip-install: -r requirements/requirements-all.txt ty==0.0.35
       # - name: Type-check with Pyright
       #   uses: jakebailey/pyright-action@v2
       #   with:
@@ -40,4 +40,4 @@ jobs:
       #     warnings: true
       - name: Type-check with ty
         run: |
-            ty check --exit-zero-on-warning --output-format=github
+            ty check --output-format=github

.github/workflows/release.yml

@@ -61,8 +61,31 @@ jobs:
             echo "should_release=false" >> $GITHUB_OUTPUT
           fi
 
+  get-version:
+    runs-on: ubuntu-slim
+    outputs:
+      ui_version: ${{ steps.version.outputs.ui_version }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - id: version
+        run: |
+          # Resolve UI version: BUILD_NUMBER from cmake/build-info.cmake > git hash + epoch > fallback
+          version=""
+          if grep -q "BUILD_NUMBER" cmake/build-info.cmake; then
+            build_number=$(grep "set(BUILD_NUMBER" cmake/build-info.cmake | grep -oP '\d+')
+            if [ -n "$build_number" ] && [ "$build_number" -gt 0 ]; then
+              version="b${build_number}"
+            fi
+          fi
+          if [ -z "$version" ]; then
+            version=$(git rev-parse --short HEAD)-$(date +%s)
+          fi
+          echo "ui_version=${version}" >> $GITHUB_OUTPUT
+
   macos-cpu:
-    needs: [check-release, ui-build]
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
     strategy:
       matrix:
@@ -96,14 +119,15 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-${{ matrix.os }}-${{ matrix.arch }}
 
@@ -117,6 +141,7 @@ jobs:
             -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
             -DLLAMA_FATAL_WARNINGS=ON \
             -DLLAMA_BUILD_BORINGSSL=ON \
+            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
@@ -142,7 +167,7 @@ jobs:
           key: release-${{ matrix.os }}-${{ matrix.arch }}
 
   ubuntu-cpu:
-    needs: [check-release, ui-build]
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
     strategy:
       matrix:
@@ -166,11 +191,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Dependencies
         id: depends
@@ -187,7 +213,7 @@ jobs:
 
       - name: ccache
         if: ${{ matrix.build != 's390x' }}
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-${{ matrix.os }}-cpu
 
@@ -201,6 +227,7 @@ jobs:
             -DGGML_NATIVE=OFF \
             -DGGML_CPU_ALL_VARIANTS=ON \
             -DLLAMA_FATAL_WARNINGS=ON \
+            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
@@ -227,7 +254,7 @@ jobs:
           key: release-${{ matrix.os }}-cpu
 
   ubuntu-vulkan:
-    needs: [check-release, ui-build]
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
     strategy:
@@ -250,11 +277,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Dependencies
         id: depends
@@ -272,7 +300,7 @@ jobs:
           fi
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-${{ matrix.os }}-vulkan
 
@@ -286,6 +314,7 @@ jobs:
             -DGGML_NATIVE=OFF \
             -DGGML_CPU_ALL_VARIANTS=ON \
             -DGGML_VULKAN=ON \
+            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
@@ -311,7 +340,7 @@ jobs:
           key: release-${{ matrix.os }}-vulkan
 
   android-arm64:
-    needs: [check-release, ui-build]
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
     runs-on: ubuntu-latest
@@ -329,11 +358,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Set up JDK
         uses: actions/setup-java@v5
@@ -358,7 +388,7 @@ jobs:
       #   cache on:  https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
       #
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.21
       #  with:
       #    key: release-android-arm64
 
@@ -377,6 +407,7 @@ jobs:
             -DLLAMA_FATAL_WARNINGS=ON \
             -DGGML_OPENMP=OFF \
             -DLLAMA_BUILD_BORINGSSL=ON \
+            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
@@ -402,7 +433,7 @@ jobs:
           name: llama-bin-android-arm64.tar.gz
 
   ubuntu-24-openvino:
-    needs: [check-release, ui-build]
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
     runs-on: ubuntu-24.04
@@ -415,8 +446,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.3.1"
-      OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Set OpenVINO version output
@@ -429,14 +460,15 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-ubuntu-24.04-openvino-release-no-preset-v1
 
@@ -476,6 +508,7 @@ jobs:
             -DGGML_OPENVINO=ON \
             -DCMAKE_INSTALL_RPATH='$ORIGIN' \
             -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
             ${{ env.CMAKE_ARGS }}
           cmake --build build/ReleaseOV --config Release --parallel
 
@@ -519,7 +552,7 @@ jobs:
           key: release-ubuntu-24.04-openvino-release-no-preset-v1
 
   windows-openvino:
-    needs: [check-release, ui-build]
+    needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
     runs-on: windows-2022
@@ -529,8 +562,8 @@ jobs:
 
     env:
       # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
-      OPENVINO_VERSION_MAJOR: "2026.3.1"
-      OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+      OPENVINO_VERSION_MAJOR: "2026.3"
+      OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
 
     steps:
       - name: Set OpenVINO version output
@@ -544,14 +577,15 @@ jobs:
         with:
             fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-openvino
           variant: ccache
@@ -648,7 +682,7 @@ jobs:
 
   windows-cpu:
     name: windows-cpu / ${{ matrix.arch }}
-    needs: [check-release, ui-build]
+    needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
     runs-on: windows-2025-vs2026
@@ -668,18 +702,19 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Install Ninja
         run: |
           choco install ninja
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
 
@@ -714,8 +749,6 @@ jobs:
         with:
           key: release-windows-2025-vs2026-${{ matrix.arch }}-cpu
 
-  # note: builds only the ggml-hip backend - llama-server is injected from the
-  #       windows-cpu zip during the release "Merge artifacts" step
   windows-rocm:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -725,7 +758,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - ROCM_VERSION: "10.0.0"
+          - ROCM_VERSION: "7.14.0"
             gpu_targets: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201"
             build: x64
 
@@ -736,12 +769,8 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install Ninja
-        run: |
-          choco install ninja
-
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
           evict-old-files: 1d
@@ -793,28 +822,33 @@ jobs:
 
       - name: Build
         run: |
-          cmake -S . -B build `
-            -G "Ninja Multi-Config" `
+          mkdir build
+          cd build
+          cmake .. `
+            -G "Unix Makefiles" `
             -DCMAKE_PREFIX_PATH="${env:HIP_PATH}" `
+            -DCMAKE_BUILD_TYPE=Release `
             -DGGML_BACKEND_DL=ON `
             -DGGML_NATIVE=OFF `
-            -DGGML_CPU=OFF `
+            -DGGML_CPU=ON `
+            -DGGML_CPU_ALL_VARIANTS=ON `
             -DGGML_HIP=ON `
             -DCMAKE_C_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
             -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang++.exe" `
             -DCMAKE_C_FLAGS="-Wno-error=incompatible-pointer-types" `
             -DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
             -DHIP_PATH="${env:HIP_PATH}" `
+            -DGGML_HIP_ROCWMMA_FATTN=ON `
             -DAMDGPU_TARGETS="${{ matrix.gpu_targets }}"
-          cmake --build build --config Release --parallel ${env:NUMBER_OF_PROCESSORS} --target ggml-hip
+          cmake --build . --config Release --parallel ${env:NUMBER_OF_PROCESSORS}
 
       - name: Verify HIP backend was built
         run: |
-          $hipDll = Get-ChildItem -Path build\bin\Release -Filter "ggml-hip*.dll" -ErrorAction SilentlyContinue
+          $hipDll = Get-ChildItem -Path build\bin -Filter "ggml-hip*.dll" -ErrorAction SilentlyContinue
           if (-not $hipDll) {
             Write-Host "##[error]ggml-hip*.dll was NOT produced. The HIP backend silently failed to build."
-            Write-Host "Contents of build\bin\Release:"
-            Get-ChildItem build\bin\Release | Format-Table -AutoSize
+            Write-Host "Contents of build\bin:"
+            Get-ChildItem build\bin | Format-Table -AutoSize
             exit 1
           }
           Write-Host "HIP backend artifact found:"
@@ -829,40 +863,10 @@ jobs:
           $rocmVersionShort = ('${{ matrix.ROCM_VERSION }}'.Split('.')[0..1] -join '.')
           echo "ROCM_VERSION_SHORT=$rocmVersionShort" >> $env:GITHUB_ENV
 
-      - name: Bundle HIP runtime DLLs (amdhip64_7.dll, rocm_kpack.dll, amd_comgr.dll)
-        run: |
-          $ErrorActionPreference = "Stop"
-          # See issue https://github.com/ggml-org/llama.cpp/issues/26929.
-          # ggml-hip.dll loads amdhip64_7.dll at run time. The Adrenalin driver
-          # ships an amdhip64_7.dll in System32, which the loader searches before PATH,
-          # so a matching DLL from PATH cannot win. Copy amdhip64 next to the
-          # binaries (exe directory is searched before System32) so the correct
-          # runtime is used. rocm_kpack.dll is amdhip64_7's direct dependency, so
-          # copy the matching version too. amd_comgr is copied as well to keep it
-          # in sync with the bundled amdhip64, avoiding a version mismatch with a
-          # amd_comgr from System32.
-          # rocblas/hipblaslt kernels resolve fine via PATH and are not copied.
-          $binPath = (rocm-sdk path --bin).Trim()
-          if (-not $binPath) { throw "rocm-sdk path --bin returned empty" }
-          write-host "ROCm bin path: $binPath"
-
-          $patterns = @("amdhip64_7.dll", "rocm_kpack.dll", "amd_comgr.dll")
-          foreach ($pattern in $patterns) {
-            $files = Get-ChildItem -Path $binPath -Filter $pattern -ErrorAction SilentlyContinue
-            if (-not $files) { throw "no match for $pattern in $binPath" }
-            foreach ($f in $files) {
-              Copy-Item $f.FullName -Destination build\bin\Release -Force
-              write-host "  copied $($f.Name)"
-            }
-          }
-
       - name: Pack artifacts
         run: |
-          7z a -snl llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip `
-            .\build\bin\Release\ggml-hip.dll `
-            .\build\bin\Release\amdhip64_7.dll `
-            .\build\bin\Release\rocm_kpack.dll `
-            .\build\bin\Release\amd_comgr.dll
+          cp "LICENSE" "build\bin\"
+          7z a -snl llama-bin-win-rocm-${{ env.ROCM_VERSION_SHORT }}-${{ matrix.build }}.zip .\build\bin\*
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v6
@@ -875,8 +879,6 @@ jobs:
         with:
           key: windows-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
 
-  # note: builds only the backend library - llama-server (with the embedded UI)
-  #       is injected from the windows-cpu zip during the release "Merge artifacts" step
   windows:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -907,6 +909,13 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
+
       - name: Install Vulkan SDK
         id: get_vulkan
         if: ${{ matrix.backend == 'vulkan' }}
@@ -923,7 +932,7 @@ jobs:
 
       # TODO: these jobs need to use llvm toolchain in order to utilize the ccache
       #- name: ccache
-      #  uses: ggml-org/ccache-action@v1.2.24
+      #  uses: ggml-org/ccache-action@v1.2.21
       #  with:
       #    key: release-windows-2025-${{ matrix.arch }}-${{ matrix.backend }}
 
@@ -969,8 +978,6 @@ jobs:
           path: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
           name: llama-bin-win-${{ matrix.backend }}-${{ matrix.arch }}.zip
 
-  # note: builds only the ggml-cuda backend - llama-server is injected from the
-  #       windows-cpu zip during the release "Merge artifacts" step
   windows-cuda:
     name: windows-cuda (${{ matrix.cuda }}, ${{ matrix.arch }})
     needs: [check-release]
@@ -999,6 +1006,13 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
+
       - name: Install Cuda Toolkit
         uses: ./.github/actions/windows-setup-cuda
         with:
@@ -1011,7 +1025,7 @@ jobs:
           choco install ninja
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
 
@@ -1070,8 +1084,6 @@ jobs:
         with:
           key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
 
-  # note: builds only the ggml-sycl backend - llama-server is injected from the
-  #       windows-cpu zip during the release "Merge artifacts" step
   windows-sycl:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
@@ -1106,8 +1118,15 @@ jobs:
           Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force
           "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
+
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-windows-2022-x64-sycl
 
@@ -1176,7 +1195,7 @@ jobs:
           key: release-windows-2022-x64-sycl
 
   ubuntu-24-sycl:
-    needs: [check-release, ui-build]
+    needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
     strategy:
@@ -1218,14 +1237,15 @@ jobs:
           wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
           sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: release-ubuntu-24.04-sycl-${{ matrix.build }}
 
@@ -1267,11 +1287,11 @@ jobs:
         with:
           key: release-ubuntu-24.04-sycl-${{ matrix.build }}
 
-  ubuntu-24-rocm:
-    needs: [check-release, ui-build]
+  ubuntu-22-rocm:
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
 
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
 
     permissions:
       actions: write
@@ -1279,7 +1299,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - ROCM_VERSION: "10.0.0"
+          - ROCM_VERSION: "7.14.0"
             gpu_targets: "gfx908;gfx90a;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1152;gfx1200;gfx1201"
             build: 'x64'
 
@@ -1290,11 +1310,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Download UI build
-        uses: actions/download-artifact@v7
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          name: llama-ui.zip
-          path: tools/ui/dist
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Free up disk space
         uses: ggml-org/free-disk-space@v1.3.1
@@ -1302,9 +1323,9 @@ jobs:
           tool-cache: true
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
-          key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
+          key: release-ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
           evict-old-files: 1d
           max-size: "1G"
 
@@ -1333,7 +1354,7 @@ jobs:
           # libraries = HIP runtime and CMake configs needed for linking
           # devel = compilers, headers, static libs
           python -m pip install --upgrade pip
-          python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
+          python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{ matrix.ROCM_VERSION }}"
 
           # Get ROCm installation paths using the rocm-sdk CLI tool
           ROCM_PATH=$(rocm-sdk path --root)
@@ -1367,6 +1388,7 @@ jobs:
             -DGPU_TARGETS="${{ matrix.gpu_targets }}" \
             -DGGML_HIP=ON \
             -DHIP_PLATFORM=amd \
+            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }} \
             ${{ env.CMAKE_ARGS }}
           cmake --build build --config Release -j $(nproc)
 
@@ -1392,10 +1414,10 @@ jobs:
       - name: ccache-clear
         uses: ./.github/actions/ccache-clear
         with:
-          key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
+          key: release-ubuntu-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
 
   ios-xcode:
-    needs: [check-release]
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
     runs-on: macos-26
 
@@ -1423,7 +1445,8 @@ jobs:
             -DLLAMA_BUILD_SERVER=OFF \
             -DCMAKE_SYSTEM_NAME=iOS \
             -DCMAKE_OSX_DEPLOYMENT_TARGET=16.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml \
+            -DHF_UI_VERSION=${{ needs.get-version.outputs.ui_version }}
           cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
 
       - name: xcodebuild for swift package
@@ -1546,9 +1569,11 @@ jobs:
 #          name: llama-bin-${{ matrix.chip_type }}-openEuler-${{ matrix.arch }}${{ matrix.use_acl_graph == 'on' && '-aclgraph' || '' }}.tar.gz
 
   ui-build:
-    needs: [check-release]
+    needs: [check-release, get-version]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}
     uses: ./.github/workflows/ui-build.yml
+    with:
+      hf_ui_version: ${{ needs.get-version.outputs.ui_version }}
 
   release:
     if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -1563,13 +1588,14 @@ jobs:
     runs-on: ubuntu-slim
 
     needs:
+      - get-version
       - windows
       - windows-cpu
       - windows-cuda
       - windows-sycl
       - windows-rocm
       - windows-openvino
-      - ubuntu-24-rocm
+      - ubuntu-22-rocm
       - ubuntu-cpu
       - ubuntu-vulkan
       - ubuntu-24-openvino
@@ -1602,27 +1628,24 @@ jobs:
           path: ./artifact
           merge-multiple: true
 
-      - name: Merge artifacts
+      - name: Move artifacts
         id: move_artifacts
         run: |
           mkdir -p release
 
-          # the windows-cpu zip contains the full toolset (llama-server with the embedded
-          # UI, ggml-cpu) - inject it into the other windows zips so that every archive
-          # ships the same binaries, only with a different backend library on top
-          echo "Injecting windows-cpu binaries (llama-server + CPU backend) into the backend zips..."
+          echo "Adding CPU backend files to existing zips..."
           for arch in x64 arm64; do
             cpu_zip="artifact/llama-bin-win-cpu-${arch}.zip"
             temp_dir=$(mktemp -d)
-            echo "Extracting windows-cpu-${arch} package..."
+            echo "Extracting CPU backend for $arch..."
             unzip "$cpu_zip" -d "$temp_dir"
 
-            echo "Merging into $arch zips..."
+            echo "Adding CPU files to $arch zips..."
             for target_zip in artifact/llama-bin-win-*-${arch}.zip; do
               if [[ "$target_zip" == "$cpu_zip" ]]; then
                 continue
               fi
-              echo "Injecting into $(basename "$target_zip")"
+              echo "Adding CPU backend to $(basename "$target_zip")"
               realpath_target_zip=$(realpath "$target_zip")
               (cd "$temp_dir" && zip -r "$realpath_target_zip" .)
             done
@@ -1646,7 +1669,7 @@ jobs:
         id: download_ui
         uses: actions/download-artifact@v7
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: ./ui-dist
 
       - name: Package UI
@@ -1703,7 +1726,7 @@ jobs:
             - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
             - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
             - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
-            - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
+            - [Ubuntu x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-7.14-x64.tar.gz)
             - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ needs.ubuntu-24-openvino.outputs.openvino_version }}-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP16)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp16-x64.tar.gz)
@@ -1717,11 +1740,11 @@ jobs:
             - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
             - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
             - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip)
-            - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
+            - [Windows arm64 (CUDA 13) (preview)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
             - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
             - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip)
             - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip)
-            - [Windows x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-10.0-x64.zip)
+            - [Windows x64 (ROCm 7.14)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-rocm-7.14-x64.zip)
 
             **openEuler:**
             - [DISABLED](https://github.com/ggml-org/llama.cpp/pull/23705)

.github/workflows/server-sanitize.yml

@@ -32,8 +32,6 @@ on:
     ]
 
 env:
-  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
-  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
   LLAMA_ARG_LOG_COLORS: 1
   LLAMA_ARG_LOG_PREFIX: 1
   LLAMA_ARG_LOG_TIMESTAMPS: 1
@@ -45,7 +43,7 @@ concurrency:
 
 jobs:
   server:
-    runs-on: hf-jobs-cpu-upgrade
+    runs-on: [self-hosted, CPU, Linux, llama-server]
 
     strategy:
       matrix:
@@ -54,6 +52,20 @@ jobs:
       fail-fast: false
 
     steps:
+      #- name: Dependencies
+      #  id: depends
+      #  run: |
+      #    sudo apt-get update
+      #    sudo apt-get -y install \
+      #      build-essential \
+      #      xxd \
+      #      git \
+      #      cmake \
+      #      curl \
+      #      wget \
+      #      language-pack-en \
+      #      libssl-dev
+
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
@@ -61,23 +73,12 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential cmake python3-full
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
         with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: server-sanitize-${{ matrix.sanitizer }}
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          node-version: "24"
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Build
         id: cmake_build
@@ -93,17 +94,9 @@ jobs:
             -DLLAMA_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }}
           cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: server-sanitize-${{ matrix.sanitizer }}
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+      - name: Python setup
+        id: setup_python
+        uses: actions/setup-python@v7
 
       - name: Install Python dependencies
         run: |
@@ -116,7 +109,8 @@ jobs:
         run: |
           source .venv/bin/activate
           cd tools/server/tests
-          PYTEST_WORKERS=1 ./tests.sh
+          export ${{ matrix.extra_args }}
+          ./tests.sh
 
       - name: Slow tests
         id: server_integration_tests_slow
@@ -124,4 +118,5 @@ jobs:
         run: |
           source .venv/bin/activate
           cd tools/server/tests
-          PYTEST_WORKERS=1 SLOW_TESTS=1 ./tests.sh
+          export ${{ matrix.extra_args }}
+          SLOW_TESTS=1 ./tests.sh

.github/workflows/server-self-hosted.yml

@@ -72,7 +72,7 @@ jobs:
         run: |
           cd tools/server/tests
           source venv/bin/activate
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx1, backend-sampling)
         id: server_integration_tests_backend_sampling
@@ -81,7 +81,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2)
         id: server_integration_tests_gpu2
@@ -90,7 +90,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_METAL_DEVICES=2
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2, backend-sampling)
         id: server_integration_tests_gpu2_backend_sampling
@@ -99,10 +99,10 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
   server-cuda:
-    runs-on: "hf-jobs-t4-small:cuda13"
+    runs-on: [self-hosted, llama-server, Linux, NVIDIA]
 
     steps:
       - name: Clone
@@ -112,42 +112,12 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y cmake libssl-dev python3 python3-venv python3-pip
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: self-hosted-server-cuda
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Build
         id: cmake_build
         run: |
-          cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+          cmake -B build -DGGML_CUDA=ON -DGGML_SCHED_NO_REALLOC=ON
           cmake --build build --config Release -j $(nproc) --target llama-server
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: self-hosted-server-cuda
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Python setup
         id: setup_python
         run: |
@@ -162,7 +132,7 @@ jobs:
         run: |
           cd tools/server/tests
           source venv/bin/activate
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx1, backend-sampling)
         id: server_integration_tests_backend_sampling
@@ -171,7 +141,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2)
         id: server_integration_tests_gpu2
@@ -180,7 +150,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_CUDA_DEVICES=2
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
       - name: Tests (GPUx2, backend-sampling)
         id: server_integration_tests_gpu2_backend_sampling
@@ -189,7 +159,7 @@ jobs:
           cd tools/server/tests
           source venv/bin/activate
           export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1
-          PYTEST_WORKERS=1 ./tests.sh
+          ./tests.sh
 
   server-kleidiai:
     runs-on: ah-ubuntu_22_04-c8g_8x

.github/workflows/server.yml

@@ -80,19 +80,11 @@ jobs:
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: server-ubuntu-24.04-arm
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: server-ubuntu-24.04-arm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+          evict-old-files: 1d
+          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
       - name: Build
         id: cmake_build
@@ -101,18 +93,6 @@ jobs:
             -DGGML_SCHED_NO_REALLOC=ON
           cmake --build build --config Release -j $(nproc) --target llama-server
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: server-ubuntu-24.04-arm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Python setup
         id: setup_python
         uses: actions/setup-python@v6
@@ -160,7 +140,7 @@ jobs:
           ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
 
       - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
+        uses: ggml-org/ccache-action@v1.2.21
         with:
           key: server-windows-2025-x64
           evict-old-files: 1d
@@ -201,13 +181,3 @@ jobs:
           cd tools/server/tests
           export SLOW_TESTS="1"
           ./tests.sh
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: server-windows-2025-x64
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/ui-build-self-hosted.yml

@@ -17,9 +17,8 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Install dependencies
         run: npm ci
@@ -32,6 +31,6 @@ jobs:
       - name: Upload built UI
         uses: actions/upload-artifact@v6
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist/
           retention-days: 1

.github/workflows/ui-build.yml

@@ -3,8 +3,8 @@ name: UI Build
 on:
   workflow_call:
     inputs:
-      ui_version:
-        description: 'Version string embedded in build.json (e.g. b1234); defaults to b<commit-count>'
+      hf_ui_version:
+        description: 'Version string for version.json (e.g. 12345)'
         required: false
         type: string
 
@@ -17,25 +17,13 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Resolve UI version
-        id: version
-        run: |
-          version="${{ inputs.ui_version }}"
-          if [ -z "$version" ]; then
-            version="b$(git rev-list --count HEAD)"
-          fi
-          echo "ui_version=${version}" >> $GITHUB_OUTPUT
 
       - name: Setup Node.js
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Install dependencies
         run: npm ci
@@ -43,7 +31,8 @@ jobs:
 
       - name: Build application
         env:
-          LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
+          HF_UI_VERSION: ${{ inputs.hf_ui_version || '' }}
+          LLAMA_BUILD_NUMBER: ${{ inputs.hf_ui_version || 'b0000' }}
         run: npm run build
         working-directory: tools/ui
 
@@ -54,6 +43,6 @@ jobs:
       - name: Upload built UI
         uses: actions/upload-artifact@v6
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist/
           retention-days: 1

.github/workflows/ui-publish.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Download UI build artifact
         uses: actions/download-artifact@v7
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist/
 
       - name: Create distribution archive

.github/workflows/ui-self-hosted.yml

@@ -64,7 +64,7 @@ jobs:
       - name: Download built UI artifacts
         uses: actions/download-artifact@v6
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist/
 
       - name: Run type checking
@@ -106,7 +106,7 @@ jobs:
       - name: Download built UI artifacts
         uses: actions/download-artifact@v6
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist/
 
       - name: Build Storybook

.github/workflows/ui.yml

@@ -57,14 +57,13 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Download built UI artifacts
         uses: actions/download-artifact@v6
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist/
 
       - name: Install dependencies
@@ -115,9 +114,8 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "24"
-          # cache: "npm"
-          # cache-dependency-path: "tools/ui/package-lock.json"
-          package-manager-cache: false
+          cache: "npm"
+          cache-dependency-path: "tools/ui/package-lock.json"
 
       - name: Install dependencies
         id: setup
@@ -128,7 +126,7 @@ jobs:
       - name: Download built UI artifacts (reuses ui-build)
         uses: actions/download-artifact@v6
         with:
-          name: llama-ui.zip
+          name: ui-build
           path: tools/ui/dist/
 
       - name: Install Playwright browsers

AUTHORS

@@ -1,4 +1,4 @@
-# date: Fri Sep  4 10:06:46 EEST 2026
+# date: Tue Aug 18 14:32:43 EEST 2026
 # this file is auto-generated by scripts/gen-authors.sh
 
 Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
@@ -46,7 +46,6 @@ Abhijit Ramesh <abhijitramesh2k@gmail.com>
 abhijitb11 <113058133+abhijitb11@users.noreply.github.com>
 Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
 Abhinay Krishna <abhinaykrishna60@gmail.com>
-Abhiram <78226909+geckguy@users.noreply.github.com>
 Abhishek Gopinath K <31348521+overtunned@users.noreply.github.com>
 abotsis <github@bots.is>
 Abraham Gonzalez <theabecaster0@gmail.com>
@@ -88,7 +87,6 @@ akleine <alb.kleine@gmx.de>
 Al G <toasting@gmail.com>
 Al Mochkin <14274697+amochkin@users.noreply.github.com>
 Alan Gray <agray3@users.noreply.github.com>
-Alan Tseng <alanhc.tseng1999@gmail.com>
 Alawode Oluwandabira <dabiraalawode@yahoo.com>
 Albert Jin <albert.jin@gmail.com>
 Alberto <57916483+albbus-stack@users.noreply.github.com>
@@ -138,9 +136,7 @@ alonfaraj <alonfaraj@gmail.com>
 AlpinDale <52078762+AlpinDale@users.noreply.github.com>
 alwqx <kenan3015@gmail.com>
 Aman <amangupta052@gmail.com>
-Aman Chadha(IVIXMMI) <79802170+ac-mmi@users.noreply.github.com>
 Aman Gupta <amangupta052@gmail.com>
-Aman Karki <itsamankarki@gmail.com>
 amd-dwang <dong.wang@amd.com>
 amd-lalithnc <lalithnc@amd.com>
 Amir <amir_zia@outlook.com>
@@ -191,7 +187,6 @@ Anton Mitkov <anton.mitkov@codeplay.com>
 Antonis Makropoulos <benuix@gmail.com>
 Anudit Nagar <nagaranudit@gmail.com>
 Anuj Attri <anujattri01@gmail.com>
-anujj <ajalota@nvidia.com>
 anzz1 <anzz1@live.com>
 Aparna M P <aparmp@qti.qualcomm.com>
 Aparna M P <quic_aparmp@quicinc.com>
@@ -201,7 +196,6 @@ arch-btw <57669023+arch-btw@users.noreply.github.com>
 arcrank <arcrank@gmail.com>
 ardfork <134447697+ardfork@users.noreply.github.com>
 Arik Poznanski <arikpoz@users.noreply.github.com>
-Aritro Bandyopadhyay <71339004+AriBandyo@users.noreply.github.com>
 arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
 Armen Kaleshian <kriation@users.noreply.github.com>
 Arsen Arutunan <58118221+limloop@users.noreply.github.com>
@@ -236,7 +230,6 @@ bandoti <141645996+bandoti@users.noreply.github.com>
 Bar Haim <barvhaim@gmail.com>
 BarfingLemurs <128182951+BarfingLemurs@users.noreply.github.com>
 Bart Louwers <bart.louwers@gmail.com>
-Bartosz Taudul <wolf@nereid.pl>
 Bartowski <3266127+bartowski1182@users.noreply.github.com>
 Bartowski <ckealty1182@gmail.com>
 Bas Nijholt <basnijholt@gmail.com>
@@ -284,7 +277,6 @@ Bono Lv <lvscar@users.noreply.github.com>
 Borislav Stanimirov <b.stanimirov@abv.bg>
 Borislav Stanimirov <b@ibob.bg>
 Bowen Han <fancycode@gmail.com>
-Brad Smith <1472326+infinitewarp@users.noreply.github.com>
 Branden Butler <bwtbutler@hotmail.com>
 Brandon Squizzato <35474886+bsquizz@users.noreply.github.com>
 Brian <mofosyne@gmail.com>
@@ -295,7 +287,6 @@ Bryan Honof <bryanhonof@gmail.com>
 bryanSwk <93190252+bryanSwk@users.noreply.github.com>
 bsilvereagle <bsilvereagle@users.noreply.github.com>
 bssrdf <merlintiger@hotmail.com>
-Buğra Özgürsoy <13810383+ozgursoy@users.noreply.github.com>
 byte-6174 <88070277+byte-6174@users.noreply.github.com>
 Caleb DeLeeuw <143902425+SolshineCode@users.noreply.github.com>
 Calvin Laurenson <calvin@laurenson.dev>
@@ -335,7 +326,6 @@ Chenguang Li <757486878@qq.com>
 Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
 Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
 chiranko <96988916+chiranko@users.noreply.github.com>
-Chris Danis <cdanis@gmail.com>
 Chris Elrod <elrodc@gmail.com>
 Chris Kuehl <ckuehl@ckuehl.me>
 Chris Lee <clee@mg8.org>
@@ -366,7 +356,6 @@ clyang <clyang@clyang.net>
 cmdr2 <secondary.cmdr2@gmail.com>
 cmdr2 <shashank.shekhar.global@gmail.com>
 cocktailpeanut <121128867+cocktailpeanut@users.noreply.github.com>
-codemonkey <441345965@qq.com>
 codezjx <code.zjx@gmail.com>
 coezbek <c.oezbek@gmail.com>
 comex <comexk@gmail.com>
@@ -378,8 +367,6 @@ Copilot <198982749+Copilot@users.noreply.github.com>
 Corentin REGAL <corentin.regal@gmail.com>
 cphlipot <9103367+cphlipot@users.noreply.github.com>
 cpumaxx <163466046+cpumaxx@users.noreply.github.com>
-cqderek <cqderek@gmail.com>
-cqderek <cqiang@qti.qualcomm.com>
 crasm <crasm@git.vczf.net>
 crasm <crasm@git.vczf.us>
 crat0z <11581854+crat0z@users.noreply.github.com>
@@ -440,7 +427,6 @@ DavidKorczynski <david@adalogics.com>
 davidrhodus <david@vacovideo.com>
 Dawid Potocki <github@dawidpotocki.com>
 Dawid Wysocki <62249621+TortillaZHawaii@users.noreply.github.com>
-Daya Adianto <addianto@users.noreply.github.com>
 ddh0 <chemist-mulches-39@icloud.com>
 ddh0 <dylanhalladay02@icloud.com>
 ddpasa <112642920+ddpasa@users.noreply.github.com>
@@ -477,7 +463,6 @@ Dmytro Romanov <casteldazur@gmail.com>
 Dobri Danchev <12420863+danchev@users.noreply.github.com>
 DocShotgun <126566557+DocShotgun@users.noreply.github.com>
 Doctor Shotgun <126566557+DocShotgun@users.noreply.github.com>
-Dominik Pantaleoni <95251853+dpantaleoni@users.noreply.github.com>
 Don Mahurin <dmahurin@users.noreply.github.com>
 Dong Won Kim <63934649+ddwkim@users.noreply.github.com>
 Donghyeon Jeong <54725479+djeong20@users.noreply.github.com>
@@ -519,7 +504,6 @@ Emmanuel Ferdman <emmanuelferdman@gmail.com>
 Emreerdog <34742675+Emreerdog@users.noreply.github.com>
 Engininja2 <139037756+Engininja2@users.noreply.github.com>
 Equim <sayaka@ekyu.moe>
-Eric A Stalee <87948564+Eric-A-Stalee@users.noreply.github.com>
 Eric Curtin <ecurtin@redhat.com>
 Eric Curtin <eric.curtin@docker.com>
 Eric Curtin <ericcurtin17@gmail.com>
@@ -535,7 +519,6 @@ Esko Toivonen <eskot98@gmail.com>
 Ethan Turner <eturner64@gmail.com>
 Ettore Di Giacinto <mudler@users.noreply.github.com>
 EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
-Eurekatic <eurekatic@eurekatic.eu>
 Evan Huus <eapache@gmail.com>
 Evan Jones <evan.q.jones@gmail.com>
 Evan Miller <emmiller@gmail.com>
@@ -694,7 +677,6 @@ HimariO <dsfhe49854@gmail.com>
 hipudding <huafengchun@gmail.com>
 Hitesh Chopra <34310832+hiteshchopra11@users.noreply.github.com>
 hksdpc255 <43977088+hksdpc255@users.noreply.github.com>
-hmirin <hmirin@users.noreply.github.com>
 hmscider <201289679+hmscider@users.noreply.github.com>
 Hoang Nguyen <hugo53@users.noreply.github.com>
 hoangmit <hoangmit@users.noreply.github.com>
@@ -719,7 +701,6 @@ Huawei Lin <huaweilin.cs@gmail.com>
 Hugo <hugo@whynothugo.nl>
 Hugo Roussel <hugo.rous@gmail.com>
 Huifeng Ou <79071290+ho2103@users.noreply.github.com>
-HumerousGorgon <31957201+HumerousGorgon@users.noreply.github.com>
 hutli <6594598+hutli@users.noreply.github.com>
 hutli <hutli@hutli.hu>
 hutli <jensstaermose@hotmail.com>
@@ -757,15 +738,12 @@ intelmatt <61025942+intelmatt@users.noreply.github.com>
 iohub <rickyang.pro@gmail.com>
 Ionoclast Laboratories <brigham@ionoclast.com>
 iron <lizhenneng@gmail.com>
-Isaac <34376531+init-22@users.noreply.github.com>
 Isaac McFadyen <isaac@imcf.me>
 IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
 Ishaan Gandhi <Ishaangandhi@gmail.com>
 iSma <ismail.senhaji@gmail.com>
 Ismail <115064057+AlrIsmail@users.noreply.github.com>
 issixx <46835150+issixx@users.noreply.github.com>
-itsnotoger <19309683+itsnotoger@users.noreply.github.com>
-itterative <190138728+itterative@users.noreply.github.com>
 Ivan <nekotekina@gmail.com>
 Ivan Chikish <nekotekina@gmail.com>
 Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
@@ -790,7 +768,6 @@ Jakkala Mahesh <155058658+MaheshJakkala@users.noreply.github.com>
 Jakub N <jakubniemczyk97@gmail.com>
 JamePeng <jame_peng@sina.com>
 James A Capozzoli <157492257+jac-jim@users.noreply.github.com>
-James Francis <6763899+JamesFranc@users.noreply.github.com>
 James O'Leary <65884233+jpohhhh@users.noreply.github.com>
 James Reynolds <magnusviri@users.noreply.github.com>
 jameswu2014 <545426914@qq.com>
@@ -821,7 +798,6 @@ Jed Fox <git@jedfox.com>
 Jeff Bolz <jbolz@nvidia.com>
 Jeffrey Morgan <jmorganca@gmail.com>
 Jeffrey Quesnelle <emozilla@nousresearch.com>
-Jeremie Miller <jeremie.miller@gmail.com>
 Jeremy Demeule <jdemeule@users.noreply.github.com>
 Jeremy Rand <244188+JeremyRand@users.noreply.github.com>
 Jeroen Mostert <jeroen.mostert@cm.com>
@@ -833,7 +809,6 @@ Jesse Jojo Johnson <williamsaintgeorge@gmail.com>
 Jesse LaRose <jesse@taey.ai>
 Jesse Posner <jesse.posner@gmail.com>
 Jesus Talavera <145992175+jesus-talavera-ibm@users.noreply.github.com>
-Jetson Tan <tanzongyouyi@outlook.com>
 Jett Janiak <jettjaniak@gmail.com>
 Jeximo <jeximo@gmail.com>
 JFLFY2255 <JFLFY2255@163.com>
@@ -850,7 +825,6 @@ Jie Fu (傅杰) <jiefu@tencent.com>
 jiez <373447296@qq.com>
 Jillis ter Hove <j.terhove@gmail.com>
 Jim Wu <jimw567@users.noreply.github.com>
-Jingxin (Philip) Li <philipaslee@gmail.com>
 Jinwoo Jeong <33892306+williamjeong2@users.noreply.github.com>
 Jinyang He <hejinyang@loongson.cn>
 jinzihao <jinzihao1996@gmail.com>
@@ -876,13 +850,11 @@ John Balis <phobossystems@gmail.com>
 John Bean <113509988+johnbean393@users.noreply.github.com>
 John Eismeier <42679190+jeis4wpi@users.noreply.github.com>
 John Smith <67539080+kingsidelee@users.noreply.github.com>
-John-Henry Lim <42513874+Interpause@users.noreply.github.com>
 Johnathan Craig Maudlin <13183098+jcmdln@users.noreply.github.com>
 JohnnyB <jboero@users.noreply.github.com>
 johnson442 <56517414+johnson442@users.noreply.github.com>
 jojorne <jojorne@users.noreply.github.com>
 jon-chuang <9093549+jon-chuang@users.noreply.github.com>
-Jonas J <111707981+John-194@users.noreply.github.com>
 Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
 Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
 Jonathan <47618606+jbuchananr@users.noreply.github.com>
@@ -952,7 +924,6 @@ Karsten Weiss <knweiss@gmail.com>
 Karthick <j.karthic2004@gmail.com>
 Karthik Kumar Viswanathan <195178+guilt@users.noreply.github.com>
 Karthik Sethuraman <k.seth1993@gmail.com>
-Kartik Gulia <kgulia@nvidia.com>
 Kartik Sirohi <99896785+sirohikartik@users.noreply.github.com>
 Kashif Rasul <kashif.rasul@gmail.com>
 KASR <karim.asrih@gmail.com>
@@ -960,7 +931,6 @@ Kasumi <90275229+kasumi-1@users.noreply.github.com>
 Katostrofik <georgiopapairo@gmail.com>
 katsu560 <118887472+katsu560@users.noreply.github.com>
 Kawrakow <48489457+ikawrakow@users.noreply.github.com>
-kbenkhaled <khalilbenkhaled01@gmail.com>
 kchro3 <62481661+kchro3@users.noreply.github.com>
 kdkd <2569413+kdkd@users.noreply.github.com>
 Keiichi Tabata <keiichi.tabata@outlook.com>
@@ -969,7 +939,6 @@ Kenvix ⭐ <kenvixzure@live.com>
 Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com>
 Kevin Gibbons <bakkot@gmail.com>
 Kevin Hannon <kehannon@redhat.com>
-Kevin Hopper <93635715+kh0pper@users.noreply.github.com>
 Kevin Ji <1146876+kevinji@users.noreply.github.com>
 Kevin Kwok <antimatter15@gmail.com>
 Kevin Liu <4396kevinliu@gmail.com>
@@ -995,14 +964,12 @@ Konstantin Herud <konstantin.herud@denkbares.com>
 Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
 Krishna Sridhar <99914379+srikris-sridhar@users.noreply.github.com>
 krystiancha <krystian@krystianch.com>
-krzsztf <krzysztof@witkowscy.org>
 kubawoo <k-wach@o2.pl>
 kumaal <44551860+kumaal@users.noreply.github.com>
 kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com>
 kunnis <kunnis@users.noreply.github.com>
 Kunshang Ji <kunshang.ji@intel.com>
 kuronekosaiko <EvanChanJ@163.com>
-kurquhar <kurquhar@qti.qualcomm.com>
 Kusha Gharahi <3326002+kushagharahi@users.noreply.github.com>
 kustaaya <58045274+kustaaya@users.noreply.github.com>
 kuvaus <22169537+kuvaus@users.noreply.github.com>
@@ -1014,7 +981,6 @@ Kyle Liang <liangmanlai@gmail.com>
 Kyle Mistele <kyle@mistele.com>
 KyleHagy <59183061+KyleHagy@users.noreply.github.com>
 Kylin <56434533+KyL0N@users.noreply.github.com>
-Kyozzz <1147385157@qq.com>
 l-austenfeld <53152202+l-austenfeld@users.noreply.github.com>
 l3utterfly <gc.pthzfoldr@gmail.com>
 l8bloom <l8bloomapi@gmail.com>
@@ -1026,7 +992,6 @@ Lars Sonchocky-Helldorf <lars.sonchocky-helldorf@hamburg.de>
 las7 <98077186+las7@users.noreply.github.com>
 Lasse Lauwerys <65569591+Iemand005@users.noreply.github.com>
 Laura <Tijntje_7@msn.com>
-Laurent Zuijdwijk <laurent.zuijdwijk@gmail.com>
 Law Po Ying <30721578+yingying0906@users.noreply.github.com>
 lcy <lcy0321@users.noreply.github.com>
 ldwang <ftgreat@163.com>
@@ -1074,8 +1039,6 @@ Ludovic Henry <git@ludovic.dev>
 Ludovic Henry <ludovic@rivosinc.com>
 Lukas Straub <lukasstraub2@web.de>
 Łukasz Ślusarczyk <112692748+lslusarczyk@users.noreply.github.com>
-Lukasz Stolcman <4583553+lstolcman@users.noreply.github.com>
-LunalFresh <165352784+LunalFresh@users.noreply.github.com>
 Luo Tian <lt@basecity.com>
 luoyu-intel <yu.luo@intel.com>
 luyhcsu <110711054+luyhcsu@users.noreply.github.com>
@@ -1091,7 +1054,6 @@ Maarten ter Huurne <maarten@treewalker.org>
 Maciej Lisowski <39798354+MaciejDromin@users.noreply.github.com>
 Mack Straight <eiz@users.noreply.github.com>
 maddes8cht <55592906+maddes8cht@users.noreply.github.com>
-Mads Marquart <mads@marquart.dk>
 Maël Kerbiriou <m431.kerbiriou@gmail.com>
 MaggotHATE <clay1326@gmail.com>
 MagicExists <106458387+gugugiyu@users.noreply.github.com>
@@ -1253,8 +1215,6 @@ Naco Siren <naco-siren@users.noreply.github.com>
 Nam D. Tran <42194884+namtranase@users.noreply.github.com>
 nanahi <130121847+na-na-hi@users.noreply.github.com>
 Nathan Epstein <nate2@umbc.edu>
-Nathan Wilson <67372905+Nathanw1014@users.noreply.github.com>
-Nathanw1014 <67372905+Nathanw1014@users.noreply.github.com>
 Natsu <chino@hotococoa.moe>
 Nauful Shaikh <nauful@gmail.com>
 NawafAlansari <72708095+NawafAlansari@users.noreply.github.com>
@@ -1277,7 +1237,6 @@ niansa/tuxifan <tuxifan@posteo.de>
 Nicholai Tukanov <nicholaitukanov@gmail.com>
 Nicholas Sparks <157740354+nisparks@users.noreply.github.com>
 Nick <0x0b4ac@gmail.com>
-Nick Farrell <nick.farrell@aiven.io>
 nick huang <nickhuang99@hotmail.com>
 Nick Lafleur <55208706+nicklafleur@users.noreply.github.com>
 Nick Towle <ntowle@gmail.com>
@@ -1300,7 +1259,6 @@ NikolaiLyssogor <59844691+NikolaiLyssogor@users.noreply.github.com>
 Nikolaos Pothitos <pothitos@di.uoa.gr>
 Nikolas <127742645+nneubacher@users.noreply.github.com>
 Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
-Nils Gladitz <nilsgladitz@gmail.com>
 Nindaleth <Nindaleth@users.noreply.github.com>
 ningshanwutuobang <ningshanwutuobang@gmail.com>
 Noah <99681487+NoahOksuz@users.noreply.github.com>
@@ -1397,7 +1355,6 @@ Pop Flamingo <trevor.annedenise@icloud.com>
 postmasters <namnguyen@google.com>
 Pouya <PooyaGhahramanian@Gmail.com>
 pqnet <119850+pqnet@users.noreply.github.com>
-Prabhsimran Singh <pskrunner14@gmail.com>
 Prabod <prabod@maincode.com>
 Prajwal B Mehendarkar <prajwal.b.mehendarkar@ibm.com>
 Pranav Dhinakar <pdhinaka@qti.qualcomm.com>
@@ -1421,7 +1378,6 @@ qouoq <qouoq@fastmail.com>
 Qu Zongfu <43257352+yancaoweidaode@users.noreply.github.com>
 quei <56998528+quei4r@users.noreply.github.com>
 Quentin Bramas <quentin.bramas@gmail.com>
-QuintinShaw <github@xyt.email>
 QuintinShaw <yx6f20@soton.ac.uk>
 qunash <anzoria@gmail.com>
 quyentonndbs <raynaedgar8677@outlook.com>
@@ -1506,7 +1462,6 @@ robertomeroni <150194833+robertomeroni@users.noreply.github.com>
 Robey Holderith <robey@flaminglunchbox.net>
 Robin Davidsson <40024429+R-Dson@users.noreply.github.com>
 Robyn <robyngraf@users.noreply.github.com>
-Rock Chen <rockchen.tw@gmail.com>
 Rőczey Barnabás <31726601+An0nie@users.noreply.github.com>
 RodriMora <bullerwins@gmail.com>
 Roger Chen <chenrui@gmail.com>
@@ -1544,21 +1499,17 @@ runfuture <runfuture@users.noreply.github.com>
 RunningLeon <maningsheng@sensetime.com>
 RunningLeon <mnsheng@yeah.net>
 Russyyds <161207317+Russyyds@users.noreply.github.com>
-Ryan C <ryan5rdx@users.noreply.github.com>
 Ryan Goulden <percontation@gmail.com>
 Ryan Landay <rlanday@gmail.com>
 Ryan Mangeno <160974989+ryan-mangeno@users.noreply.github.com>
 Ryder Wishart <ryderwishart@gmail.com>
 Ryuei <louixs@users.noreply.github.com>
 s-goto-11 <206795233+s-goto-11@users.noreply.github.com>
-s0mecode <213953308+s0mecode@users.noreply.github.com>
 s8322 <s0527684199@gmail.com>
-Saad Ali <NIXKnight@users.noreply.github.com>
 Saba Fallah <10401143+sfallah@users.noreply.github.com>
 Saba Fallah <sabafallah@gmail.com>
 Sachin Desai <smdesai@gmail.com>
 Sachin Sharma <sachin@zettabolt.com>
-Safi Ullah <safiullah.3915@gmail.com>
 safranowith <bsh155762@gmail.com>
 SakuraUmi <yukinon244@gmail.com>
 Salvador E. Tropea <stropea@inti.gob.ar>
@@ -1601,7 +1552,6 @@ Sergey Alirzaev <l29ah@riseup.net>
 Sergey Alirzaev <zl29ah@gmail.com>
 Sergey Fedorov <vital.had@gmail.com>
 Sergey Malinin <sergmalinin@gmail.com>
-Sergey Sklyarov <sergey.sklyarov@gmail.com>
 Sergio López <slp@redhat.com>
 Sergio López <slp@sinrega.org>
 Sergiu <8598216+mzsergiu@users.noreply.github.com>
@@ -1632,13 +1582,11 @@ Shawn Gu <shawngu@qti.qualcomm.com>
 Shawn yang <137684499+Yangxiaoz@users.noreply.github.com>
 Shelby Jenkins <47464908+ShelbyJenkins@users.noreply.github.com>
 Sheldon Robinson <sheldon.robinson@live.com>
-Shenghan Yang <ysharke@sjtu.edu.cn>
 shibe2 <shibe@tuta.io>
 Shijie <821898965@qq.com>
 Shin-myoung-serp <relent95@naver.com>
 Shintarou Okada <kokuzen@gmail.com>
 shivamkumard-ctrl <shivamkumard@nvidia.com>
-Shobhit <sobhit.me@gmail.com>
 Shouyu <65317431+joeldushouyu@users.noreply.github.com>
 Shouzheng Liu <61452103+lshzh-ww@users.noreply.github.com>
 Shouzheng Liu <lshzh.hi@gmail.com>
@@ -1659,7 +1607,6 @@ Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
 Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
 simevo <github@simevo.com>
 Simon Redman <simon@ergotech.com>
-Simon Teixidor <simon@flaskpost.me>
 Simon Willison <swillison@gmail.com>
 simon886212 <37953122+simon886212@users.noreply.github.com>
 Simranjeet Singh <105192966+simrnsingh@users.noreply.github.com>
@@ -1716,7 +1663,6 @@ stevenkuang <stevenkuang@tencent.com>
 Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
 StrangeBytesDev <141275258+StrangeBytesDev@users.noreply.github.com>
 strawberrymelonpanda <152940198+strawberrymelonpanda@users.noreply.github.com>
-Strongtut <Strongtut@users.noreply.github.com>
 Suaj Carrot <72162667+SuajCarrot@users.noreply.github.com>
 sudhiarm <sudhi.sathyavathy@arm.com>
 Sukriti Sharma <Ssukriti@users.noreply.github.com>
@@ -1741,7 +1687,6 @@ Tamar <Tamar0812@outlook.co.il>
 tamarPal <tamarp3385@gmail.com>
 Tameem <113388789+AhmadTameem@users.noreply.github.com>
 Tamotsu Takahashi <ttakah+github@gmail.com>
-Tanner Bruhn <66120666+tannerbruhn@users.noreply.github.com>
 tarcey <cey.tarik@gmail.com>
 Tarek Dakhran <t.dakhran@gmail.com>
 Tarek Dakhran <tarek@liquid.ai>
@@ -1751,7 +1696,6 @@ Taylor <quantumtraveling@gmail.com>
 tc-mb <157115220+tc-mb@users.noreply.github.com>
 TecJesh <qdvm5gl@163.com>
 Tei Home <taiteitonghome@proton.me>
-Tekin Ertekin <tekin.ertekin@gmail.com>
 Tekin Ertekin <tekinertekin@gmail.com>
 tempstudio <49735574+tempstudio@users.noreply.github.com>
 teo <TeoZosa@users.noreply.github.com>
@@ -1793,7 +1737,6 @@ Ting Lou <louting@189.cn>
 Ting Lou <ting.lou@gmail.com>
 Ting Sun <suntcrick@gmail.com>
 Titaniumtown <titaniumtown@proton.me>
-Tiwei Bie <tiwei.btw@antgroup.com>
 tjohnman <tjohnman@users.noreply.github.com>
 Tobias Lütke <tobi@shopify.com>
 Toby <25832191+aetherbird@users.noreply.github.com>
@@ -1870,7 +1813,6 @@ Vishal Agarwal <vishalagarwal.jss@gmail.com>
 Vishal Singh <vishal@zettabolt.com>
 Vitali Lovich <vlovich+github@gmail.com>
 Vivian <vynride@gmail.com>
-vk <89937361+itsvedantkumar@users.noreply.github.com>
 Vlad <spitfireage@gmail.com>
 Vladimir <bogdad@gmail.com>
 Vladimir Malyutin <first-leon@yandex.ru>
@@ -1955,7 +1897,6 @@ Yaiko <elyaiko@hotmail.com>
 Yakine Tahtah <96926916+ReinforcedKnowledge@users.noreply.github.com>
 YangLe <smilingpoplar@gmail.com>
 yangli2 <yangli2@gmail.com>
-Yaniss Amazouz <yaniss91600@gmail.com>
 Yann Follet <131855179+YannFollet@users.noreply.github.com>
 Yanzhao Wang <yanzhaow@qti.qualcomm.com>
 Yarden Tal <yardent@qti.qualcomm.com>

CMakeLists.txt

@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)
 
 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 4)
+set(LLAMA_VERSION_MINOR 2)
 set(LLAMA_VERSION_PATCH 0)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
 
@@ -134,8 +134,8 @@ option(LLAMA_BUILD_TOOLS     "llama: build tools"
 option(LLAMA_BUILD_EXAMPLES  "llama: build examples"                                                             ${LLAMA_STANDALONE})
 option(LLAMA_BUILD_SERVER    "llama: build server example"                                                       ${LLAMA_STANDALONE})
 option(LLAMA_BUILD_APP       "llama: build the unified binary"                                                   ${LLAMA_STANDALONE})
-option(LLAMA_BUILD_UI        "llama: build the embedded Web UI for server"                                       OFF)
-option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
+option(LLAMA_BUILD_UI        "llama: build the embedded Web UI for server"                                       ON)
+option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available (requires LLAMA_BUILD_UI=ON)" ON)
 
 option(LLAMA_TOOLS_INSTALL "llama: install tools" ${LLAMA_TOOLS_INSTALL_DEFAULT})
 option(LLAMA_TESTS_INSTALL "llama: install tests" ON)
@@ -197,16 +197,6 @@ llama_option_depr(WARNING     LLAMA_CURL)
 include("cmake/license.cmake")
 license_add_file("llama.cpp" "LICENSE")
 
-#
-# compile options
-#
-
-# clang stores the modification time of the precompiled header sources inside the
-# header and rejects it when they differ, so the timestamp is left out of it
-add_compile_options(
-    "$<$<COMPILE_LANG_AND_ID:C,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>"
-    "$<$<COMPILE_LANG_AND_ID:CXX,Clang,IntelLLVM>:SHELL:-Xclang -fno-pch-timestamp>")
-
 #
 # 3rd-party
 #

CODEOWNERS

@@ -57,7 +57,6 @@
 /ggml/src/ggml-cann/                    @ggml-org/ggml-cann
 /ggml/src/ggml-common.h                 @ggerganov
 /ggml/src/ggml-cpu/                     @ggerganov
-/ggml/src/ggml-cpu/iqp.*                @bartowski1182
 /ggml/src/ggml-cpu/spacemit/            @alex-spacemit
 /ggml/src/ggml-cuda/                    @ggml-org/ggml-cuda
 /ggml/src/ggml-cuda/vendors/hip.h       @IMbackK

CONTRIBUTING.md

@@ -74,7 +74,6 @@ For more info, please refer to the [AGENTS.md](AGENTS.md) file.
 - If a PR does not warrant a new release, add `[no release]` in the squashed commit to spare CI resources
 - Be mindful of maintenance: most of the work going into a feature happens after the PR is merged. If the PR author is not committed to contribute long-term, someone else needs to take responsibility (you)
 - Add the ["merge ready"](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr+is%3Aopen+draft%3Ano+sort%3Aupdated-desc+label%3A%22merge+ready%22+) label to a PR to indicate when a PR can be fast-merged without waiting for 2 independent reviews. [(more info)](https://github.com/ggml-org/llama.cpp/pull/26178)
-- Wait for CI results before merging
 
 Maintainers reserve the right to decline review or close pull requests for any reason, without any questions, particularly under any of the following conditions:
 - The proposed change is already mentioned in the roadmap or an existing issue, and it has been assigned to someone.

README.md

@@ -13,7 +13,7 @@
 [![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
 [![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
 
-[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
+[manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3A0cc4m%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [compile times](https://github.com/ggml-org/llama.cpp-dev/blob/master/README-compile-times.md) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
 
 </div>
 
@@ -74,7 +74,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 | [CANN](docs/build.md#cann) | Ascend NPU |
 | [CUDA](docs/build.md#cuda) | Nvidia GPU |
 | [HIP](docs/build.md#hip) | AMD GPU |
-| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
+| [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon |
 | [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
 | [MUSA](docs/build.md#musa) | Moore Threads GPU |
 | [Metal](docs/build.md#metal-build) | Apple Silicon |

app/llama.cpp

@@ -80,7 +80,7 @@ static const command cmds[] = {
 #undef UPDATE_HIDDEN
 
 static int version(int /*argc*/, char ** /*argv*/) {
-    llama_print_build_info(llama_version(), stdout);
+    llama_print_build_info(llama_version());
     return 0;
 }
 

build-xcframework.sh

@@ -18,7 +18,7 @@ LLAMA_BUILD_TESTS=OFF
 LLAMA_BUILD_SERVER=OFF
 LLAMA_BUILD_MTMD=ON
 GGML_METAL=ON
-GGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY:-ON}
+GGML_METAL_EMBED_LIBRARY=ON
 GGML_BLAS_DEFAULT=ON
 GGML_OPENMP=OFF
 
@@ -169,14 +169,6 @@ setup_framework_structure() {
     cp tools/mtmd/mtmd.h           ${header_path}
     cp tools/mtmd/mtmd-helper.h    ${header_path}
 
-    if [[ "$GGML_METAL_EMBED_LIBRARY" == "OFF" ]]; then
-        if [[ "$platform" == "macos" ]]; then
-            cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/Versions/A/Resources/
-        else
-            cp ${build_dir}/bin/*.metallib ${build_dir}/framework/${framework_name}.framework/
-        fi
-    fi
-
     # Create module map (common for all platforms)
     cat > ${module_path}module.modulemap << EOF
 framework module llama {
@@ -458,7 +450,6 @@ build_ios_sim() {
         -DIOS=ON \
         -DCMAKE_SYSTEM_NAME=iOS \
         -DCMAKE_OSX_SYSROOT=iphonesimulator \
-        -DGGML_METAL_TARGET_OS=ios \
         -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
@@ -476,7 +467,6 @@ build_ios_device() {
         -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \
         -DCMAKE_SYSTEM_NAME=iOS \
         -DCMAKE_OSX_SYSROOT=iphoneos \
-        -DGGML_METAL_TARGET_OS=ios \
         -DCMAKE_OSX_ARCHITECTURES="arm64" \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
@@ -508,7 +498,6 @@ build_visionos() {
         -DCMAKE_OSX_ARCHITECTURES="arm64" \
         -DCMAKE_SYSTEM_NAME=visionOS \
         -DCMAKE_OSX_SYSROOT=xros \
-        -DGGML_METAL_TARGET_OS=xros \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
         -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
@@ -527,7 +516,6 @@ build_visionos_sim() {
         -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
         -DCMAKE_SYSTEM_NAME=visionOS \
         -DCMAKE_OSX_SYSROOT=xrsimulator \
-        -DGGML_METAL_TARGET_OS=xros \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \
         -DCMAKE_C_FLAGS="${COMMON_C_FLAGS}" \
         -DCMAKE_CXX_FLAGS="${COMMON_CXX_FLAGS}" \
@@ -546,7 +534,6 @@ build_tvos_sim() {
         -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
         -DCMAKE_SYSTEM_NAME=tvOS \
         -DCMAKE_OSX_SYSROOT=appletvsimulator \
-        -DGGML_METAL_TARGET_OS=tvos \
         -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
         -DGGML_METAL=ON \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \
@@ -565,7 +552,6 @@ build_tvos_device() {
         -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \
         -DCMAKE_SYSTEM_NAME=tvOS \
         -DCMAKE_OSX_SYSROOT=appletvos \
-        -DGGML_METAL_TARGET_OS=tvos \
         -DCMAKE_OSX_ARCHITECTURES="arm64" \
         -DGGML_METAL=ON \
         -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \

ci/run.sh

@@ -189,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
     fi
     CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
 
-    # TODO: fix failing tests on OpenVINO backend
-    CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-|test-save-load-state"
+    # TODO: fix and re-enable the `test-llama-archs` test below
+    CTEST_EXTRA="-E test-llama-archs|test-recurrent-state-rollback-nemotron-h"
 fi
 
 ## helpers
@@ -250,7 +250,7 @@ function gg_run_ctest_debug {
     (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
     (time cmake --build . --config Debug -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
 
-    (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
+    (time ctest -C Debug --output-on-failure -L main -E "test-opt|test-backend-ops|test-llama-archs" ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
 
     set +e
 }
@@ -300,69 +300,6 @@ function gg_sum_ctest_release {
     gg_printf '```\n'
 }
 
-# test_llama_archs_tensor_split
-
-function gg_run_test_llama_archs_tensor_split {
-    cd ${SRC}
-
-    set -e
-
-    if [ ! -z ${GG_BUILD_CUDA} ]; then
-        GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-        GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-        GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-        GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-    fi
-
-    if [ ! -z ${GG_BUILD_METAL} ]; then
-        GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-        GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-        GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-        GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
-    fi
-
-    set +e
-}
-
-function gg_sum_test_llama_archs_tensor_split {
-    gg_printf '### %s\n\n' "${ci}"
-
-    gg_printf 'Runs test-llama-archs with 1 to 4 devices\n'
-    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
-    gg_printf '```\n'
-    gg_printf '%s\n' "$(cat $OUT/${ci}.log)"
-    gg_printf '```\n'
-}
-
-# test_llama_archs_models
-
-function gg_run_test_llama_archs_models {
-    cd ${SRC}
-
-    set -e
-
-    # TODO: fix and re-enable `test-llama-archs` on OpenVINO
-    # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
-    if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
-        rm -rf build-ci-models && mkdir -p build-ci-models
-
-        # generate the dummy models used by the model-dependent tests
-        ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
-    fi
-
-    set +e
-}
-
-function gg_sum_test_llama_archs_models {
-    gg_printf '### %s\n\n' "${ci}"
-
-    gg_printf 'Generates the dummy models used by the model-dependent tests\n'
-    gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
-    gg_printf '```\n'
-    gg_printf '%s\n' "$(cat $OUT/${ci}.log)"
-    gg_printf '```\n'
-}
-
 # test_scripts
 
 function gg_run_test_scripts {
@@ -761,55 +698,27 @@ function gg_check_build_requirements {
         gg_printf 'ctest not found, please install\n'
         exit 1
     fi
-
-    if ! command -v unzip &> /dev/null; then
-        gg_printf 'unzip not found, please install\n'
-        exit 1
-    fi
 }
 
-function gg_run_test_backend_ops {
+function gg_run_test_backend_ops_cpu {
     cd ${SRC}
 
     cd build-ci-release
 
     set -e
 
-    local n_jobs=$(nproc)
-    if [ "${n_jobs}" -gt 2 ]; then
-        n_jobs=2
-    fi
-    local args_extra="-j ${n_jobs}"
-
-    # TODO: fix multi-threaded for ROCm
-    #       https://github.com/ggml-org/llama.cpp/actions/runs/34576278519/job/103297889044?pr=28740#step:3:4865
-    if [ ! -z ${GG_BUILD_ROCM} ]; then
-        args_extra=""
-    fi
-
-    # TODO: MoltenVK bug?
-    #       https://github.com/ggml-org/llama.cpp/actions/runs/34611260059/job/103302413736?pr=28740#step:3:5897
-    if [ ! -z "${GG_BUILD_VULKAN}" ] && [ "$(uname -s)" = "Darwin" ]; then
-        args_extra=""
-    fi
-
-    # TODO: reduce the test-backend-ops timeout to 1800s
-    if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
-        (time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
-    else
-        (time timeout 3600 ./bin/test-backend-ops ${args_extra}       ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
-    fi
+    (time ./bin/test-backend-ops -b CPU ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops-cpu.log
 
     set +e
 }
 
-function gg_sum_test_backend_ops {
+function gg_sum_test_backend_ops_cpu {
     gg_printf '### %s\n\n' "${ci}"
 
-    gg_printf 'Runs test-backend-ops\n'
+    gg_printf 'Runs test-backend-ops for CPU backend\n'
     gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
     gg_printf '```\n'
-    gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops.log)"
+    gg_printf '%s\n' "$(cat $OUT/${ci}-test-backend-ops-cpu.log)"
     gg_printf '```\n'
     gg_printf '\n'
 }
@@ -842,10 +751,9 @@ ret=0
 test $ret -eq 0 && gg_run ctest_debug
 test $ret -eq 0 && gg_run ctest_release
 
-test $ret -eq 0 && gg_run test_backend_ops
-
-test $ret -eq 0 && gg_run test_llama_archs_models
-test $ret -eq 0 && gg_run test_llama_archs_tensor_split
+if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
+    test $ret -eq 0 && gg_run test_backend_ops_cpu
+fi
 
 if [ -z ${GG_BUILD_LOW_PERF} ]; then
     test $ret -eq 0 && gg_run embd_bge_small

common/CMakeLists.txt

@@ -53,10 +53,7 @@ endif()
 
 set(TARGET llama-common)
 
-include(parsers/sources.cmake)
-
 add_library(${TARGET}
-    ${LLAMA_CHAT_PARSERS_SOURCES}
     arg.cpp
     arg.h
     base64.hpp
@@ -84,8 +81,6 @@ add_library(${TARGET}
     imatrix-loader.cpp
     imatrix-loader.h
     json-schema-to-grammar.cpp
-    json-schema.cpp
-    json-schema.h
     json.cpp
     json.h
     llguidance.cpp
@@ -136,8 +131,6 @@ set_target_properties(${TARGET} PROPERTIES
 target_include_directories(${TARGET} PUBLIC .)
 target_link_libraries     (${TARGET} PUBLIC vendor::nlohmann vendor::sheredom)
 target_compile_features   (${TARGET} PUBLIC cxx_std_17)
-target_precompile_headers (${TARGET} PRIVATE common.h)
-target_precompile_headers (${TARGET} PRIVATE chat.h)
 
 if (LLAMA_SUBPROCESS)
     target_compile_definitions(${TARGET} PUBLIC LLAMA_SUBPROCESS)

common/arg.cpp

@@ -872,6 +872,17 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
                     arg.c_str(), e.what(), opt.to_string().c_str()));
             }
         }
+
+        // TODO: remove this check after deprecating --mmap|mlock|dio
+        auto has_arg = [&](std::initializer_list<const char *> names) {
+            return std::any_of(names.begin(), names.end(), [&](const char * name) {
+                return seen_args.count(name);
+            });
+        };
+        if (has_arg({"-lm", "--load-mode"}) &&
+            has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) {
+            LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n");
+        }
     };
 
     // parse all CLI args now, so that -hf is available below for remote preset resolution
@@ -883,12 +894,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
     postprocess_cpu_params(params.speculative.draft.cpuparams,       &params.cpuparams);
     postprocess_cpu_params(params.speculative.draft.cpuparams_batch, &params.cpuparams_batch);
 
-    // default the mmproj device to the global device selection if not set explicitly with -mmdev
-    if (params.mmproj_use_gpu && params.mmproj_device == nullptr && !params.devices.empty()) {
-        params.mmproj_device = params.devices.front();
-        params.mmproj_use_gpu = params.mmproj_device != nullptr;
-    }
-
     if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {
         throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
     }
@@ -955,11 +960,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
         ));
     }
 
-    // if the preserve_reasoning kwarg was not specified explicitly, enable it by default
-    if (!params.default_template_kwargs.count("preserve_reasoning")) {
-        params.default_template_kwargs["preserve_reasoning"] = "true";
-    }
-
     return true;
 }
 
@@ -1643,14 +1643,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
         }
     ).set_env("LLAMA_ARG_CTX_SIZE"));
-    add_opt(common_arg(
-        { "--kv-unified-per-slot" }, "N",
-        "context limit per parallel slot (default: unset, behavior unchanged).\n"
-        "when set without -c/--ctx-size, the shared KV pool is sized to n_parallel*N",
-        [](common_params & params, int value) {
-            params.kv_unified_per_slot = value;
-        }
-    ).set_env("LLAMA_ARG_KV_UNIFIED_PER_SLOT").set_examples({ LLAMA_EXAMPLE_SERVER }));
     add_opt(common_arg(
         {"-n", "--predict", "--n-predict"}, "N",
         string_format(
@@ -2277,14 +2269,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_sampling());
     add_opt(common_arg(
         {"-j", "--json-schema"}, "SCHEMA",
-        "JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
+        "JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
         [](common_params & params, const std::string & value) {
             params.sampling.grammar = {COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, json_schema_to_grammar(json::parse(value))};
         }
     ).set_sampling());
     add_opt(common_arg(
         {"-jf", "--json-schema-file"}, "FILE",
-        "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{\"type\": \"object\"}` for any JSON object",
+        "File containing a JSON schema to constrain generations (https://json-schema.org/), e.g. `{}` for any JSON object\nFor schemas w/ external $refs, use --grammar + example/json_schema_to_grammar.py instead",
         [](common_params & params, const std::string & value) {
             std::ifstream file(value);
             if (!file) {
@@ -2605,7 +2597,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     add_opt(common_arg(
         // note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
         {"-mmdev", "--mmproj-device"}, "DEVICE",
-        "device to use for multimodal projector (none = don't offload, default: follows --device)\n"
+        "device to use for multimodal projector (none = don't offload, default: auto)\n"
         "use --list-devices to see a list of available devices",
         [](common_params & params, const std::string & value) {
             if (value == "none") {
@@ -2652,27 +2644,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.mtmd_batch_max_tokens = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
-    add_opt(common_arg(
-        {"--video-fps"}, "N",
-        string_format("target video frame rate (default: %.1f)", params.video_fps),
-        [](common_params & params, const std::string & value) {
-            params.video_fps = std::stof(value);
-        }
-    ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FPS"));
-    add_opt(common_arg(
-        {"--video-timestamp-interval"}, "N",
-        string_format("interval in milliseconds between text timestamps (default: %" PRId64 ")", params.video_timestamp_interval_ms),
-        [](common_params & params, int value) {
-            params.video_timestamp_interval_ms = value;
-        }
-    ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"));
-    add_opt(common_arg(
-        {"--video-ffmpeg-dir"}, "DIR",
-        "path to the directory containing ffmpeg and ffprobe (default: search in PATH)",
-        [](common_params & params, const std::string & value) {
-            params.video_ffmpeg_bin_dir = value;
-        }
-    ).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));
     if (params.is_gen_docs || llama_supports_rpc()) {
         add_opt(common_arg(
             {"--rpc"}, "SERVERS",
@@ -2683,6 +2654,32 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             }
         ).set_env("LLAMA_ARG_RPC"));
     }
+    add_opt(common_arg(
+        {"--mlock"},
+        "DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing",
+        [](common_params & params) {
+            LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");
+            params.load_mode = LLAMA_LOAD_MODE_MLOCK;
+        }
+    ).set_env("LLAMA_ARG_MLOCK"));
+    add_opt(common_arg(
+        {"--mmap"},
+        {"--no-mmap"},
+        "DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)",
+        [](common_params & params, bool value) {
+            LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n");
+            params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE;
+        }
+    ).set_env("LLAMA_ARG_MMAP"));
+    add_opt(common_arg(
+        {"-dio", "--direct-io"},
+        {"-ndio", "--no-direct-io"},
+        "DEPRECATED in favor of `--load-mode`: use DirectIO if available",
+        [](common_params & params, bool value) {
+            LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n");
+            params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE;
+        }
+    ).set_env("LLAMA_ARG_DIO"));
     add_opt(common_arg(
         {"-lm", "--load-mode"}, "MODE",
         "model loading mode (default: auto)\n"
@@ -2702,19 +2699,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             else { throw std::invalid_argument("invalid value"); }
         }
     ).set_env("LLAMA_ARG_LOAD_MODE"));
-    add_opt(common_arg(
-        {"-lzm", "--lazy-mode"}, "MODE",
-        "on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
-        "- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
-        "- auto: on, but only for tensors larger than 4 GiB\n"
-        "- off: always keep them resident",
-        [](common_params & params, const std::string & value) {
-            /**/ if (value == "on")   { params.lazy_mode = LLAMA_LAZY_MODE_ON;   }
-            else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
-            else if (value == "off")  { params.lazy_mode = LLAMA_LAZY_MODE_OFF;  }
-            else { throw std::invalid_argument("invalid value"); }
-        }
-    ).set_env("LLAMA_ARG_LAZY_MODE"));
     add_opt(common_arg(
         {"--numa"}, "TYPE",
         "attempt optimizations that help on some NUMA systems\n"
@@ -2766,20 +2750,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             if (value < 0) {
                 throw std::invalid_argument("invalid value");
             }
-            llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides);
+            for (int i = 0; i < value; ++i) {
+                // keep strings alive and avoid leaking memory by storing them in a static vector
+                static std::list<std::string> buft_overrides;
+                buft_overrides.push_back(llm_ffn_exps_block_regex(i));
+                params.tensor_buft_overrides.push_back({buft_overrides.back().c_str(), ggml_backend_cpu_buffer_type()});
+            }
         }
     ).set_env("LLAMA_ARG_N_CPU_MOE"));
-    add_opt(common_arg(
-        {"-ncffn", "--n-cpu-ffn"}, "N",
-        "keep the dense FFN weights of the first N layers in the CPU\n"
-        "(dense models; for MoE expert weights use --n-cpu-moe)",
-        [](common_params & params, int value) {
-            if (value < 0) {
-                throw std::invalid_argument("invalid value");
-            }
-            llm_add_n_cpu_ffn_overrides(value, LLM_FFN_DENSE_REGEX, params.tensor_buft_overrides);
-        }
-    ).set_env("LLAMA_ARG_N_CPU_FFN"));
     GGML_ASSERT(params.n_gpu_layers < 0); // string_format would need to be extended for a default >= 0
     add_opt(common_arg(
         {"-ngl", "--gpu-layers", "--n-gpu-layers"}, "N",
@@ -3527,10 +3505,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
                     LOG_WRN("Setting 'enable_thinking' via --chat-template-kwargs is deprecated. "
                             "Use --reasoning on / --reasoning off instead.\n");
                 }
-                if (item.key() == "preserve_reasoning") {
-                    LOG_WRN("Setting 'preserve_reasoning' via --chat-template-kwargs is deprecated. "
-                            "Use --reasoning-preserve / --no-reasoning-preserve instead.\n");
-                }
                 params.default_template_kwargs[item.key()] = item.value().dump();
             }
         }
@@ -3721,7 +3695,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     add_opt(common_arg(
         {"--reasoning-preserve"},
         {"--no-reasoning-preserve"},
-        "preserve reasoning trace in the full history, not just the last assistant message (default: enabled)\n"
+        "preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n"
         "compatible with certain templates having 'supports_preserve_reasoning' capability\n"
         "example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking",
         [](common_params & params, bool value) {
@@ -3730,7 +3704,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             } else {
                 params.default_template_kwargs["preserve_reasoning"] = "false";
             }
-            params.preserve_reasoning_specified = true;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE"));
     add_opt(common_arg(
@@ -3870,14 +3843,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             common_log_set_file(common_log_main(), value.c_str());
         }
     ).set_env("LLAMA_ARG_LOG_FILE"));
-    add_opt(common_arg(
-        {"--log-jsonl"},
-        {"--no-log-jsonl"},
-        "Log as JSONL (one JSON object per line) to stdout, this also disables colored logging (default: disabled)",
-        [](common_params &, bool value) {
-            common_log_set_jsonl(value);
-        }
-    ).set_env("LLAMA_ARG_LOG_JSONL"));
     add_opt(common_arg(
         {"--log-prompts-dir"}, "PATH",
         "Log prompts to directory (auto-created if not present; only used for debugging, default: disabled)",
@@ -4119,7 +4084,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             if (value < 0) {
                 throw std::invalid_argument("invalid value");
             }
-            llm_add_n_cpu_ffn_overrides(value, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides);
+            for (int i = 0; i < value; ++i) {
+                static std::list<std::string> buft_overrides_draft;
+                buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i));
+                params.speculative.draft.tensor_buft_overrides.push_back({buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()});
+            }
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE"));
 
@@ -4140,38 +4109,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.speculative.draft.n_min = value;
         }
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
-    add_opt(common_arg(
-        {"--spec-synth-len"}, "L",
-        "target mean synthetic acceptance length, including the target token (benchmarking only)",
-        [](common_params & params, const std::string & value) {
-            const std::string text = string_strip(value);
-            size_t pos = 0;
-            const double length = std::stod(text, &pos);
-            if (pos != text.size() || length == -1.0) {
-                throw std::invalid_argument("invalid value");
-            }
-            params.speculative.synth_len = length;
-        }
-    ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN"));
-    add_opt(common_arg(
-        {"--spec-synth-rates"}, "P0,P1,...",
-        "comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)",
-        [](common_params & params, const std::string & value) {
-            const auto values = string_split<std::string>(value, ',');
-            std::vector<double> rates;
-            rates.reserve(values.size());
-            for (const auto & raw : values) {
-                const std::string text = string_strip(raw);
-                size_t pos = 0;
-                const double rate = std::stod(text, &pos);
-                if (pos != text.size()) {
-                    throw std::invalid_argument("invalid value");
-                }
-                rates.push_back(rate);
-            }
-            params.speculative.synth_rates = std::move(rates);
-        }
-    ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES"));
 
     add_opt(common_arg(
         {"--spec-draft-p-split", "--draft-p-split"}, "P",
@@ -4198,7 +4135,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING"));
     add_opt(common_arg(
         {"--spec-draft-device", "-devd", "--device-draft"}, "<dev1,dev2,..>",
-        "comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)\n"
+        "comma-separated list of devices to use for offloading the draft model (none = don't offload)\n"
         "use --list-devices to see a list of available devices",
         [](common_params & params, const std::string & value) {
             params.speculative.draft.devices = parse_device_list(value);

common/build-info.cpp.in

@@ -29,7 +29,7 @@ const char * llama_build_info(void) {
     return s.c_str();
 }
 
-void llama_print_build_info(const char * llama_version, FILE * stream) {
-    fprintf(stream, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
-    fprintf(stream, "built with %s for %s\n", llama_compiler(), llama_build_target());
+void llama_print_build_info(const char * llama_version) {
+    fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
+    fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target());
 }

common/build-info.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <cstdio>
-
 int llama_build_number(void);
 
 const char * llama_commit(void);
@@ -10,4 +8,4 @@ const char * llama_compiler(void);
 const char * llama_build_target(void);
 const char * llama_build_info(void);
 
-void llama_print_build_info(const char *, FILE * = stderr);
+void llama_print_build_info(const char *);

common/chat-auto-parser-generator.cpp

@@ -5,7 +5,6 @@
 #include "common.h"
 #include "json-schema-to-grammar.h"
 #include "log.h"
-#include "parsers/parsers.h"
 #include "peg-parser.h"
 
 #include <stdexcept>
@@ -13,6 +12,16 @@
 
 using json = common_json;
 
+// Helper to iterate over tools/functions
+static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
+    for (const auto & tool : tools) {
+        if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
+            continue;
+        }
+        fn(tool);
+    }
+}
+
 namespace autoparser {
 
 parser_build_context::parser_build_context(common_chat_peg_builder & p, const generation_params & inputs) :
@@ -78,6 +87,15 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
     if (include_grammar) {
         data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
         data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
+            foreach_function(inputs.tools, [&](const json & tool) {
+                const auto & function = tool.at("function");
+                auto         schema   = function.contains("parameters") ? function.at("parameters") : json::object();
+                builder.resolve_refs(schema);
+            });
+            if (has_response_format) {
+                auto schema = inputs.json_schema;
+                builder.resolve_refs(schema);
+            }
             parser.build_grammar(builder, data.grammar_lazy);
         });
 
@@ -294,7 +312,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_json(parser_build_context
     foreach_function(inputs.tools, [&](const json & tool) {
         const auto & func   = tool.at("function");
         std::string  name   = func.at("name");
-        const auto   schema = common_chat_tool_parameters(func);
+        const auto & schema = func.contains("parameters") ? func.at("parameters") : json::object();
 
         // Build call_id parser based on position (if supported)
         bool have_call_id = false;
@@ -365,31 +383,43 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
     common_peg_parser tool_choice = p.choice();
 
     foreach_function(inputs.tools, [&](const json & tool) {
-        const auto & func = tool.at("function");
-        std::string  name = func.at("name");
+        const auto &          func       = tool.at("function");
+        std::string           name       = func.at("name");
+        auto                  params     = func.contains("parameters") ? func.at("parameters") : json::object();
+        const auto &          properties = params.contains("properties") ? params.at("properties") : json::object();
+
+        std::set<std::string> required;
+        if (params.contains("required")) {
+            required = params.at("required").get<std::set<std::string>>();
+        }
+
+        auto schema_info = common_schema_info();
+        schema_info.resolve_refs(params);
 
         // Build parser for each argument, separating required and optional
         std::vector<common_peg_parser> required_parsers;
         std::vector<common_peg_parser> optional_parsers;
-        foreach_parameter(func, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
+        for (const auto & [param_name, param_schema] : properties.items()) {
+            bool is_required = required.find(param_name) != required.end();
+
             auto arg =
-                p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param.name)) +
+                p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) +
                                            arguments.name_suffix) +
                            arguments.value_prefix +
-                           (param.schema->may_be_string() ?
+                           (schema_info.resolves_to_string(param_schema) ?
                                 p.ac(p.tool_arg_string_value(until_suffix) +
                                     p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) :
                                 (p.tool_arg_json_value(p.schema(
-                                    p.json(), "tool-" + name + "-arg-" + param.name + "-schema", doc, *param.schema)) +
+                                    p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) +
                                     p.tool_arg_close(p.literal(arguments.value_suffix)))));
 
-            auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
-            if (param.required) {
+            auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg);
+            if (is_required) {
                 required_parsers.push_back(named_arg);
             } else {
                 optional_parsers.push_back(named_arg);
             }
-        });
+        }
 
         // Build required arg sequence in definition order
         common_peg_parser args_seq = p.eps();

common/chat-peg-parser.cpp

@@ -488,7 +488,7 @@ common_peg_parser common_chat_peg_builder::standard_constructed_tools(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         // Build argument parsers
         auto args = eps();
@@ -565,7 +565,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         auto args = eps();
         if (params.contains("properties") && !params["properties"].empty()) {
@@ -640,7 +640,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         // Build inner object fields
         std::vector<common_peg_parser> inner_fields;
@@ -726,7 +726,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +
                           atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
@@ -795,7 +795,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
         }
         const auto &   function = tool_def.at("function");
         std::string    name     = function.at("name");
-        ordered_json   params   = common_chat_tool_parameters(function);
+        ordered_json   params   = function.contains("parameters") ? function.at("parameters") : ordered_json::object();
 
         auto tool_name_ = name_key_parser + space() + literal(":") + space() +
                          atomic(literal("\"") + tool_name(literal(name)) + literal("\""));

common/chat.h

@@ -360,9 +360,6 @@ common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg
 
 common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
 
-// The parameters schema of a function tool. A tool without parameters, or with an empty {}, takes zero arguments.
-common_json common_chat_tool_parameters(const common_json & function);
-
 // get template caps, useful for reporting to server /props endpoint
 std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
 

common/common.cpp

@@ -402,11 +402,10 @@ void common_params_print_info(const common_params & params, bool print_devices)
 #endif
     COM_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);
 
-    const int verbosity = common_log_get_verbosity_thold();
-    COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, verbosity);
+    COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, common_log_get_verbosity_thold());
 
     // device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device
-    if (print_devices && verbosity >= LOG_LEVEL_TRACE) {
+    if (print_devices) {
         COM_TRC("%s", "device_info:\n");
         for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
             auto * dev = ggml_backend_dev_get(i);
@@ -1688,7 +1687,6 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
     mparams.main_gpu        = params.main_gpu;
     mparams.split_mode      = params.split_mode;
     mparams.load_mode       = params.load_mode;
-    mparams.lazy_mode = params.lazy_mode;
     mparams.tensor_split    = params.tensor_split;
     mparams.check_tensors   = params.check_tensors;
     mparams.use_extra_bufts = !params.no_extra_bufts;

common/common.h

@@ -8,7 +8,6 @@
 #include "ggml.h"
 #include "llama.h"
 
-#include <list>
 #include <set>
 #include <sstream>
 #include <string>
@@ -270,7 +269,7 @@ struct common_params_sampling {
         COMMON_SAMPLER_TYPE_TEMPERATURE,
     };
 
-    common_grammar                      grammar;          // optional grammar constraint (user / output-format / tool-calls)
+    common_grammar              grammar;      // optional grammar constraint (user / output-format / tool-calls)
     bool                                grammar_lazy = false;
     std::vector<common_grammar_trigger> grammar_triggers; // optional triggers (for lazy grammars)
     std::set<llama_token>               preserved_tokens;
@@ -370,9 +369,6 @@ struct common_params_speculative_ngram_cache {
 struct common_params_speculative {
     std::vector<enum common_speculative_type> types = { COMMON_SPECULATIVE_TYPE_NONE };
 
-    double synth_len = -1.0;
-    std::vector<double> synth_rates;
-
     // used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model
     common_params_speculative_draft draft;
 
@@ -387,10 +383,6 @@ struct common_params_speculative {
         return !draft.mparams.empty();
     }
 
-    bool has_synth() const {
-        return synth_len != -1.0 || !synth_rates.empty();
-    }
-
     uint32_t need_n_rs_seq() const {
         bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
             return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
@@ -483,8 +475,6 @@ struct common_params {
     enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
     enum llama_load_mode  load_mode  = LLAMA_LOAD_MODE_AUTO; // how to load the model
 
-    enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch
-
     common_cpu_params cpuparams;
     common_cpu_params cpuparams_batch;
 
@@ -599,11 +589,6 @@ struct common_params {
     int image_max_tokens = -1;
     int mtmd_batch_max_tokens = 1024;
 
-    // for video input
-    float       video_fps                   = 4.0f;
-    int64_t     video_timestamp_interval_ms = 5000;
-    std::string video_ffmpeg_bin_dir        = "";
-
     // finetune
     struct lr_opt lr;
     enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;
@@ -627,7 +612,6 @@ struct common_params {
     bool    cache_prompt        = true;  // whether to enable prompt caching
     bool    cache_idle_slots    = true;  // save and clear idle slots upon starting a new task
     int32_t n_ctx_checkpoints   = 32;    // max number of context checkpoints per slot
-    int32_t kv_unified_per_slot = 0;     // max context per parallel slot; 0 = unset
     int32_t checkpoint_min_step = 8192;  // minimum spacing between context checkpoints
     int32_t cache_ram_mib       = 8192;  // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
 
@@ -657,7 +641,6 @@ struct common_params {
     std::string ssl_file_cert = "";                                                                         // NOLINT
 
     std::map<std::string, std::string> default_template_kwargs;
-    bool preserve_reasoning_specified = false;
 
     // CLI params
     std::string server_base; // if set, connect to this server instead of starting a new one
@@ -1125,30 +1108,19 @@ const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
 }
 
 //
-// FFN offload utils
+// MoE utils
 //
 
 const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate|gate_up)_(ch|)exps";
 
-const char * const LLM_FFN_DENSE_REGEX = "\\.ffn_(up|down|gate)\\.";
-
-inline std::string llm_ffn_block_regex(int idx, const char * ffn_regex) {
-    return string_format("blk\\.%d%s", idx, ffn_regex);
+inline std::string llm_ffn_exps_block_regex(int idx) {
+    return string_format("blk\\.%d%s", idx, LLM_FFN_EXPS_REGEX);
 }
 
 inline llama_model_tensor_buft_override llm_ffn_exps_cpu_override() {
     return { LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() };
 }
 
-inline void llm_add_n_cpu_ffn_overrides(int n, const char * ffn_regex, std::vector<llama_model_tensor_buft_override> & overrides) {
-    // keep strings alive and avoid leaking memory by storing them in a static list
-    static std::list<std::string> buft_override_strings;
-    for (int i = 0; i < n; ++i) {
-        buft_override_strings.push_back(llm_ffn_block_regex(i, ffn_regex));
-        overrides.push_back({buft_override_strings.back().c_str(), ggml_backend_cpu_buffer_type()});
-    }
-}
-
 //
 // training utils
 //

common/fit.cpp

@@ -1,6 +1,5 @@
 #include "fit.h"
 
-#include "json.h"
 #include "log.h"
 
 #include "../src/llama-ext.h"
@@ -916,9 +915,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
 
     std::vector<std::array<std::string, 9>> table_data;
     table_data.reserve(devices.size());
-
-    // same data as the table below, for --log-jsonl consumers
-    common_json rows = common_json::array();
     const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
     const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
     const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
@@ -993,19 +989,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             std::to_string(mb.context / MiB),
             std::to_string(mb.compute / MiB),
             std::to_string(unaccounted / static_cast<int64_t>(MiB))});
-
-        rows.push_back({
-            {"kind",        "device"},
-            {"name",        name},
-            {"description", desc},
-            {"total",       total / MiB},
-            {"free",        free / MiB},
-            {"self",        self / MiB},
-            {"model",       mb.model / MiB},
-            {"context",     mb.context / MiB},
-            {"compute",     mb.compute / MiB},
-            {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
-        });
     }
 
     // print memory breakdown for host:
@@ -1021,15 +1004,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             std::to_string(mb_host.context / MiB),
             std::to_string(mb_host.compute / MiB),
             ""}); // unaccounted
-
-        rows.push_back({
-            {"kind",    "host"},
-            {"name",    "Host"},
-            {"self",    self / MiB},
-            {"model",   mb_host.model / MiB},
-            {"context", mb_host.context / MiB},
-            {"compute", mb_host.compute / MiB},
-        });
     }
 
     // print memory breakdown for all remaining buffer types:
@@ -1051,16 +1025,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             std::to_string(mb.context / MiB),
             std::to_string(mb.compute / MiB),
             ""}); // unaccounted
-
-        rows.push_back({
-            {"kind",    "buffer_type"},
-            {"name",    name},
-            {"self",    self / MiB},
-            {"model",   mb.model / MiB},
-            {"context", mb.context / MiB},
-            {"compute", mb.compute / MiB},
-        });
-
         seen_buffer_types.insert(buft);
     }
 
@@ -1078,11 +1042,6 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
             __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),
             td[6].c_str(), td[7].c_str(), td[8].c_str());
     }
-
-    LOG_JSON("fit_memory_breakdown", common_json({
-        {"unit", "MiB"},
-        {"rows", rows},
-    }));
 }
 
 void common_fit_print(

common/jinja/caps.cpp

@@ -117,7 +117,6 @@ caps caps_get(jinja::program & prog) {
 
     JJ_DEBUG("%s\n", ">>> Running capability check: typed content");
 
-    bool checks_for_string = false;
     static const std::string content_marker = "STRING_MARKER";
 
     // case: typed content support
@@ -137,10 +136,6 @@ caps caps_get(jinja::program & prog) {
         [&](context &, bool success, value & messages, value &, const std::string & rendered) {
             auto & content = messages->at(0)->at("content");
             caps_print_stats(content, "messages[0].content");
-            if (has_op(content, "test_is_string")) {
-                // checked if content is string
-                checks_for_string = true;
-            }
             bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
             if (used_as_array) {
                 // accessed as an array
@@ -156,33 +151,6 @@ caps caps_get(jinja::program & prog) {
         }
     );
 
-    if (checks_for_string) {
-        caps_try_execute(
-            prog,
-            [&]() {
-                // messages
-                return json::array({
-                    {
-                        {"role", "user"},
-                        {"content", json::array({
-                        })}
-                    }
-                });
-            },
-            nullptr, // ctx_fn
-            nullptr, // tools_fn
-            [&](context &, bool success, value & messages, value &, const std::string &) {
-                auto & content = messages->at(0)->at("content");
-                caps_print_stats(content, "messages[0].content");
-                bool used_as_array = has_op(content, "selectattr") || has_op(content, "array_access");
-                if (used_as_array && success) {
-                    // accessed as an array
-                    result.supports_typed_content = true;
-                }
-            }
-        );
-    }
-
     JJ_DEBUG("%s\n", ">>> Running capability check: system prompt");
 
     // case: system prompt support

common/jinja/runtime.cpp

@@ -167,12 +167,6 @@ value binary_expression::execute_impl(context & ctx) {
         }
         throw std::runtime_error("Cannot perform operation " + op.value + " on undefined values");
     } else if (is_val<value_none>(left_val) || is_val<value_none>(right_val)) {
-        if (!is_val<value_none>(right_val) && (op.value == "in" || op.value == "not in")) {
-            // case: none in {'low': 1}
-            // A null left operand is looked up like any other value.
-            bool member = test_is_in();
-            return mk_val<value_bool>(op.value == "in" ? member : !member);
-        }
         if (op.value == "+" || op.value == "~") {
             value res = mk_val<value_undefined>();
             if (workaround_concat_null_with_str(res)) {
@@ -418,18 +412,12 @@ value test_expression::execute_impl(context & ctx) {
         throw std::runtime_error("Invalid test expression");
     }
 
-    const std::string test_name = "test_is_" + test_id;
-    auto it = builtins.find(test_name);
-    JJ_DEBUG("Test expression %s '%s' %s (using function '%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_name.c_str());
+    auto it = builtins.find("test_is_" + test_id);
+    JJ_DEBUG("Test expression %s '%s' %s (using function 'test_is_%s')", operand->type().c_str(), test_id.c_str(), negate ? "(negate)" : "", test_id.c_str());
     if (it == builtins.end()) {
         throw std::runtime_error("Unknown test '" + test_id + "'");
     }
 
-    if (ctx.is_get_stats) {
-        value_t::stats_t::mark_used(input);
-        input->stats.ops.insert(test_name);
-    }
-
     auto res = it->second(args);
 
     if (negate) {
@@ -842,12 +830,6 @@ value member_expression::execute_impl(context & ctx) {
         } else {
             property = this->property->execute(ctx);
         }
-    } else if (is_stmt<integer_literal>(this->property)) {
-        // syntax: obj.index
-        property = mk_val<value_int>(cast_stmt<integer_literal>(this->property)->val);
-        if (property->as_int() < 0) {
-            throw std::runtime_error("Static member property cannot be negative");
-        }
     } else {
         // syntax: obj.prop
         if (!is_stmt<identifier>(this->property)) {

common/json-schema-to-grammar.cpp

@@ -1,7 +1,5 @@
 #include "json-schema-to-grammar.h"
 #include "common.h"
-#include "trie.h"
-#include "unicode.h"
 
 #include <algorithm>
 #include <limits>
@@ -338,20 +336,18 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
     return 2 + n_hex;
 }
 
-class common_chat_schema_converter {
+class common_schema_converter {
 private:
+    friend class common_schema_info;
     friend std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options);
+    std::function<json(const std::string &)> _fetch_json;
     bool _dotall;
     std::map<std::string, std::string> _rules;
+    std::unordered_map<std::string, json> _refs;
     std::unordered_set<std::string> _refs_being_resolved;
     std::vector<std::string> _errors;
     std::vector<std::string> _warnings;
 
-    template <typename T>
-    static const T & as(const common_chat_schema & node) {
-        return static_cast<const T &>(node);
-    }
-
     std::string _add_rule(const std::string & name, const std::string & rule) {
         std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
         if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
@@ -367,11 +363,11 @@ private:
         return key;
     }
 
-    std::string _generate_union_rule(const std::string & name, const std::vector<common_chat_schema_ptr> & alt_schemas) {
+    std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
         std::vector<std::string> rules;
         rules.reserve(alt_schemas.size());
         for (size_t i = 0; i < alt_schemas.size(); i++) {
-            rules.push_back(visit(*alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
+            rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
         }
         return string_join(rules, " | ");
     }
@@ -638,68 +634,85 @@ private:
             -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["]
     */
     std::string _not_strings(const std::vector<std::string> & strings) {
-        common_trie trie(strings);
+
+        struct TrieNode {
+            std::map<char, TrieNode> children;
+            bool is_end_of_string;
+
+            TrieNode() : is_end_of_string(false) {}
+
+            void insert(const std::string & string) {
+                auto *node = this;
+                for (char c : string) {
+                    node = &node->children[c];
+                }
+                node->is_end_of_string = true;
+            }
+        };
+
+        TrieNode trie;
+        for (const auto & s : strings) {
+            trie.insert(s);
+        }
 
         std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
         std::ostringstream out;
         out << "[\"] ( ";
-        std::function<void(size_t)> visit = [&](size_t idx) {
-            const auto & node = trie.nodes[idx];
-            std::string rejects;
+        std::function<void(const TrieNode &)> visit = [&](const TrieNode & node) {
+            std::ostringstream rejects;
             auto first = true;
-            for (const auto & [cpt, child] : node.children) {
-                std::string c = common_unicode_cpt_to_utf8(cpt);
-                rejects += c;
+            for (const auto & kv : node.children) {
+                rejects << kv.first;
                 if (first) {
                     first = false;
                 } else {
                     out << " | ";
                 }
-                out << "[" << c << "]";
-                if (!trie.nodes[child].children.empty()) {
+                out << "[" << kv.first << "]";
+                if (!kv.second.children.empty()) {
                     out << " (";
-                    visit(child);
+                    visit(kv.second);
                     out << ")";
-                } else {
+                } else if (kv.second.is_end_of_string) {
                     out << " " << char_rule << "+";
                 }
             }
             if (!node.children.empty()) {
-                out << " | [^\"" << rejects << "] " << char_rule << "*";
+                if (!first) {
+                    out << " | ";
+                }
+                out << "[^\"" << rejects.str() << "] " << char_rule << "*";
             }
         };
-        visit(0);
+        visit(trie);
 
         out << " )";
-        if (trie.nodes[0].pattern < 0) {
+        if (!trie.is_end_of_string) {
             out << "?";
         }
         out << " [\"]";
         return out.str();
     }
 
-    std::string _resolve_ref(const common_chat_schema_ref & schema) {
-        auto it = schema.ref.find('#');
-        std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref;
+    std::string _resolve_ref(const std::string & ref) {
+        auto it = ref.find('#');
+        std::string ref_fragment = it != std::string::npos ? ref.substr(it + 1) : ref;
         static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
         std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-");
-        if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(schema.ref) == _refs_being_resolved.end()) {
-            if (!schema.target) {
-                _errors.push_back("Unresolved $ref " + schema.ref);
-                return "";
-            }
-            _refs_being_resolved.insert(schema.ref);
-            ref_name = visit(*schema.target, ref_name);
-            _refs_being_resolved.erase(schema.ref);
+        if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {
+            _refs_being_resolved.insert(ref);
+            json resolved = _refs[ref];
+            ref_name = visit(resolved, ref_name);
+            _refs_being_resolved.erase(ref);
         }
         return ref_name;
     }
 
     std::string _build_object_rule(
-        const std::vector<std::pair<std::string, const common_chat_schema *>> & properties,
+        const std::vector<std::pair<std::string, json>> & properties,
         const std::unordered_set<std::string> & required,
         const std::string & name,
-        const common_chat_schema * additional_properties)
+        const json & additional_properties)
     {
         std::vector<std::string> required_props;
         std::vector<std::string> optional_props;
@@ -709,7 +722,7 @@ private:
             const auto &prop_name = kv.first;
             const auto &prop_schema = kv.second;
 
-            std::string prop_rule_name = visit(*prop_schema, name + (name.empty() ? "" : "-") + prop_name);
+            std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
             prop_kv_rule_names[prop_name] = _add_rule(
                 name + (name.empty() ? "" : "-") + prop_name + "-kv",
                 format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
@@ -721,10 +734,10 @@ private:
             }
             prop_names.push_back(prop_name);
         }
-        if (additional_properties) {
+        if ((additional_properties.is_boolean() && additional_properties.get<bool>()) || additional_properties.is_object()) {
             std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
             std::string value_rule =
-                additional_properties->kind() != common_chat_schema::KIND_ANY ? visit(*additional_properties, sub_name + "-value")
+                additional_properties.is_object() ? visit(additional_properties, sub_name + "-value")
                 : _add_primitive("value", PRIMITIVE_RULES.at("value"));
 
             auto key_rule =
@@ -735,10 +748,6 @@ private:
             optional_props.push_back("*");
         }
 
-        if (required_props.empty() && optional_props.empty()) {
-            return "\"{\" space \"}\"";
-        }
-
         std::string rule = "\"{\" space ";
         for (size_t i = 0; i < required_props.size(); i++) {
             if (i > 0) {
@@ -812,163 +821,267 @@ private:
     }
 
 public:
-    explicit common_chat_schema_converter(bool dotall) : _dotall(dotall) {
+    common_schema_converter(
+        const std::function<json(const std::string &)> & fetch_json,
+        bool dotall)
+          : _fetch_json(fetch_json), _dotall(dotall)
+    {
         _rules["space"] = SPACE_RULE;
     }
 
-    std::string add_schema(const std::string & name, const common_chat_schema & schema) {
-        return visit(schema, name);
+    void resolve_refs(json & schema, const std::string & url) {
+        /*
+        * Resolves all $ref fields in the given schema, fetching any remote schemas,
+        * replacing each $ref with absolute reference URL and populates _refs with the
+        * respective referenced (sub)schema dictionaries.
+        */
+        std::function<void(json &)> visit_refs = [&](json & n) {
+            if (n.is_array()) {
+                for (auto & x : n) {
+                    visit_refs(x);
+                }
+            } else if (n.is_object()) {
+                if (n.contains("$ref")) {
+                    std::string ref = n["$ref"];
+                    if (_refs.find(ref) == _refs.end()) {
+                        json target;
+                        if (ref.find("https://") == 0) {
+                            std::string base_url = ref.substr(0, ref.find('#'));
+                            auto it = _refs.find(base_url);
+                            if (it != _refs.end()) {
+                                target = it->second;
+                            } else {
+                                // Fetch the referenced schema and resolve its refs
+                                auto referenced = _fetch_json(ref);
+                                resolve_refs(referenced, base_url);
+                                _refs[base_url] = referenced;
+                            }
+                            if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) {
+                                return;
+                            }
+                        } else if (ref.find("#/") == 0) {
+                            target = schema;
+                            n["$ref"] = url + ref;
+                            ref = url + ref;
+                        } else {
+                            _errors.push_back("Unsupported ref: " + ref);
+                            return;
+                        }
+                        std::string pointer = ref.substr(ref.find('#') + 1);
+                        std::vector<std::string> tokens = string_split(pointer, "/");
+                        for (size_t i = 1; i < tokens.size(); ++i) {
+                            const std::string& sel = tokens[i];
+                            if (target.is_object() && target.contains(sel)) {
+                                target = target[sel];
+                            } else if (target.is_array()) {
+                                size_t sel_index;
+                                try {
+                                    sel_index = std::stoull(sel);
+                                } catch (const std::invalid_argument & e) {
+                                    sel_index = target.size();
+                                }
+                                if (sel_index >= target.size()) {
+                                    _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
+                                    return;
+                                }
+                                target = target[sel_index];
+                            } else {
+                                _errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
+                                return;
+                            }
+                        }
+                        _refs[ref] = target;
+                    }
+                } else {
+                    for (const auto & kv : n.items()) {
+                        visit_refs(kv.value());
+                    }
+                }
+            }
+        };
+
+        visit_refs(schema);
     }
 
     static std::string _generate_constant_rule(const json & value) {
         return format_literal(value.dump());
     }
 
-    std::string _visit_primitive(const std::string & rule_name, const std::string & type) {
-        return _add_primitive(rule_name == "root" ? "root" : type, PRIMITIVE_RULES.at(type));
-    }
-
-    std::string _visit_all_of(const common_chat_schema_all_of & schema, const std::string & name, const std::string & rule_name) {
-        std::unordered_set<std::string> required;
-        std::vector<std::pair<std::string, const common_chat_schema *>> properties;
-        std::map<std::string, size_t> enum_values;
-        std::function<void(const common_chat_schema &, bool)> add_component = [&](const common_chat_schema & comp, bool is_required) {
-            if (comp.kind() == common_chat_schema::KIND_REF) {
-                if (const auto * target = as<common_chat_schema_ref>(comp).target) {
-                    add_component(*target, is_required);
-                }
-            } else if (comp.kind() == common_chat_schema::KIND_OBJECT) {
-                for (const auto & prop : as<common_chat_schema_object>(comp).properties) {
-                    properties.emplace_back(prop.name, prop.schema.get());
-                    if (is_required) {
-                        required.insert(prop.name);
-                    }
-                }
-            } else if (comp.kind() == common_chat_schema::KIND_ENUM) {
-                for (const auto & v : as<common_chat_schema_enum>(comp).values) {
-                    enum_values[_generate_constant_rule(v)] += 1;
-                }
-            }
-        };
-        for (const auto & child : schema.children) {
-            if (child->kind() == common_chat_schema::KIND_ANY_OF) {
-                for (const auto & alt : as<common_chat_schema_any_of>(*child).children) {
-                    add_component(*alt, false);
-                }
-            } else {
-                add_component(*child, true);
-            }
-        }
-        if (!enum_values.empty()) {
-            std::vector<std::string> enum_intersection;
-            for (const auto & p : enum_values) {
-                if (p.second == schema.children.size()) {
-                    enum_intersection.push_back(p.first);
-                }
-            }
-            if (!enum_intersection.empty()) {
-                return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
-            }
-        }
-        return _add_rule(rule_name, _build_object_rule(properties, required, name, nullptr));
-    }
-
-    std::string visit(const common_chat_schema & schema, const std::string & name) {
+    std::string visit(const json & schema, const std::string & name) {
+        json schema_type = schema.contains("type") ? schema["type"] : json();
+        std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : "";
         std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
-        std::string sub_name  = name + (name.empty() ? "" : "-");
 
-        switch (schema.kind()) {
-            case common_chat_schema::KIND_REF:
-                return _add_rule(rule_name, _resolve_ref(as<common_chat_schema_ref>(schema)));
-            case common_chat_schema::KIND_ANY_OF:
-                return _add_rule(rule_name, _generate_union_rule(name, as<common_chat_schema_any_of>(schema).children));
-            case common_chat_schema::KIND_ALL_OF:
-                return _visit_all_of(as<common_chat_schema_all_of>(schema), name, rule_name);
-            case common_chat_schema::KIND_CONST:
-                return _add_rule(rule_name, _generate_constant_rule(as<common_chat_schema_const>(schema).value));
-            case common_chat_schema::KIND_ENUM: {
-                std::vector<std::string> enum_values;
-                for (const auto & v : as<common_chat_schema_enum>(schema).values) {
-                    enum_values.push_back(_generate_constant_rule(v));
-                }
-                return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
+        if (schema.contains("$ref")) {
+            return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
+        }
+        if (schema.contains("oneOf") || schema.contains("anyOf")) {
+            const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf");
+            std::vector<json> alt_schemas;
+            for (const auto & alt : alts) {
+                alt_schemas.push_back(alt);
             }
-            case common_chat_schema::KIND_OBJECT: {
-                const auto & obj = as<common_chat_schema_object>(schema);
-                if (obj.properties.empty() && obj.additional_properties && obj.additional_properties->kind() == common_chat_schema::KIND_ANY) {
-                    return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
-                }
-                std::vector<std::pair<std::string, const common_chat_schema *>> properties;
-                std::unordered_set<std::string> required;
-                for (const auto & prop : obj.properties) {
-                    properties.emplace_back(prop.name, prop.schema.get());
-                    if (prop.required) {
-                        required.insert(prop.name);
+            return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
+        }
+        if (schema_type.is_array()) {
+            std::vector<json> schema_types;
+            for (const auto & t : schema_type) {
+                json schema_copy(schema);
+                schema_copy["type"] = t;
+                schema_types.push_back(schema_copy);
+            }
+            return _add_rule(rule_name, _generate_union_rule(name, schema_types));
+        }
+        if (schema.contains("const")) {
+            return _add_rule(rule_name, _generate_constant_rule(schema["const"]));
+        }
+        if (schema.contains("enum")) {
+            std::vector<std::string> enum_values;
+            for (const auto & v : schema["enum"]) {
+                enum_values.push_back(_generate_constant_rule(v));
+            }
+            return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")");
+        }
+        if ((schema_type.is_null() || schema_type == "object")
+                && (schema.contains("properties") ||
+                    (schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
+            std::unordered_set<std::string> required;
+            if (schema.contains("required") && schema["required"].is_array()) {
+                for (const auto & item : schema["required"]) {
+                    if (item.is_string()) {
+                        required.insert(item.get<std::string>());
                     }
                 }
-                return _add_rule(rule_name, _build_object_rule(properties, required, name, obj.additional_properties.get()));
             }
-            case common_chat_schema::KIND_TUPLE: {
-                const auto & items = as<common_chat_schema_tuple>(schema).items;
+            std::vector<std::pair<std::string, json>> properties;
+            if (schema.contains("properties")) {
+                for (const auto & prop : schema["properties"].items()) {
+                    properties.emplace_back(prop.key(), prop.value());
+                }
+            }
+            return _add_rule(rule_name,
+                _build_object_rule(
+                    properties, required, name,
+                    schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
+        }
+        if ((schema_type.is_null() || schema_type == "object" || schema_type == "string") && schema.contains("allOf")) {
+            std::unordered_set<std::string> required;
+            std::vector<std::pair<std::string, json>> properties;
+            std::map<std::string, size_t> enum_values;
+            const std::string& hybrid_name = name;
+            std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) {
+                if (comp_schema.contains("$ref")) {
+                    add_component(_refs[comp_schema["$ref"]], is_required);
+                } else if (comp_schema.contains("properties")) {
+                    for (const auto & prop : comp_schema["properties"].items()) {
+                        properties.emplace_back(prop.key(), prop.value());
+                        if (is_required) {
+                            required.insert(prop.key());
+                        }
+                    }
+                } else if (comp_schema.contains("enum")) {
+                    for (const auto & v : comp_schema["enum"]) {
+                        const auto rule = _generate_constant_rule(v);
+                        if (enum_values.find(rule) == enum_values.end()) {
+                            enum_values[rule] = 0;
+                        }
+                        enum_values[rule] += 1;
+                    }
+                } else {
+                  // todo warning
+                }
+            };
+            for (const auto & t : schema["allOf"]) {
+                if (t.contains("anyOf")) {
+                    for (const auto & tt : t["anyOf"]) {
+                        add_component(tt, false);
+                    }
+                } else {
+                    add_component(t, true);
+                }
+            }
+            if (!enum_values.empty()) {
+                std::vector<std::string> enum_intersection;
+                for (const auto & p : enum_values) {
+                    if (p.second == schema["allOf"].size()) {
+                        enum_intersection.push_back(p.first);
+                    }
+                }
+                if (!enum_intersection.empty()) {
+                    return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")");
+                }
+            }
+            return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
+        }
+        if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
+            json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
+            if (items.is_array()) {
                 std::string rule = "\"[\" space ";
                 for (size_t i = 0; i < items.size(); i++) {
                     if (i > 0) {
                         rule += " \",\" space ";
                     }
-                    rule += visit(*items[i], sub_name + "tuple-" + std::to_string(i));
+                    rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
                 }
                 rule += " space \"]\"";
                 return _add_rule(rule_name, rule);
             }
-            case common_chat_schema::KIND_ARRAY: {
-                const auto & arr = as<common_chat_schema_array>(schema);
-                if (arr.items->kind() == common_chat_schema::KIND_ANY && arr.min_items == 0 && arr.max_items < 0) {
-                    return _visit_primitive(rule_name, "array");
-                }
-                std::string item_rule_name = visit(*arr.items, sub_name + "item");
-                int max_items = arr.max_items < 0 ? std::numeric_limits<int>::max() : arr.max_items;
-                return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, arr.min_items, max_items, "\",\" space") + " space \"]\"");
-            }
-            case common_chat_schema::KIND_STRING: {
-                const auto & str = as<common_chat_schema_string>(schema);
-                if (!str.pattern.empty()) {
-                    return _visit_pattern(str.pattern, rule_name);
-                }
-                if (str.format == common_chat_schema::FORMAT_UUID) {
-                    return _visit_primitive(rule_name, "uuid");
-                }
-                if (str.format != common_chat_schema::FORMAT_NONE) {
-                    std::string prim_name = std::string(str.format == common_chat_schema::FORMAT_DATE ? "date" : str.format == common_chat_schema::FORMAT_TIME ? "time" : "date-time") + "-string";
-                    return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
-                }
-                if (str.min_length > 0 || str.max_length >= 0) {
-                    std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
-                    int max_len = str.max_length < 0 ? std::numeric_limits<int>::max() : str.max_length;
-                    return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, str.min_length, max_len) + " \"\\\"\"");
-                }
-                return _visit_primitive(rule_name, "string");
-            }
-            case common_chat_schema::KIND_INTEGER: {
-                const auto & i = as<common_chat_schema_integer>(schema);
-                if (i.minimum == std::numeric_limits<int64_t>::min() && i.maximum == std::numeric_limits<int64_t>::max()) {
-                    return _visit_primitive(rule_name, "integer");
-                }
-                std::stringstream out;
-                out << "(";
-                build_min_max_int(i.minimum, i.maximum, out);
-                out << ")";
-                return _add_rule(rule_name, out.str());
-            }
-            case common_chat_schema::KIND_NUMBER:
-                return _visit_primitive(rule_name, "number");
-            case common_chat_schema::KIND_BOOLEAN:
-                return _visit_primitive(rule_name, "boolean");
-            case common_chat_schema::KIND_NULL:
-                return _visit_primitive(rule_name, "null");
-            case common_chat_schema::KIND_ANY:
-                return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
+            std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
+            int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
+            json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
+            int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
+
+            return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\"");
         }
-        return "";
+        if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
+            return _visit_pattern(schema["pattern"], rule_name);
+        }
+        if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
+            return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
+        }
+        if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
+            auto prim_name = schema_format + "-string";
+            return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
+        }
+        if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
+            std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
+            int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
+            int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
+            return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\"");
+        }
+        if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
+            int64_t min_value = std::numeric_limits<int64_t>::min();
+            int64_t max_value = std::numeric_limits<int64_t>::max();
+            if (schema.contains("minimum")) {
+                min_value = schema["minimum"].get<int64_t>();
+            } else if (schema.contains("exclusiveMinimum")) {
+                min_value = schema["exclusiveMinimum"].get<int64_t>() + 1;
+            }
+            if (schema.contains("maximum")) {
+                max_value = schema["maximum"].get<int64_t>();
+            } else if (schema.contains("exclusiveMaximum")) {
+                max_value = schema["exclusiveMaximum"].get<int64_t>() - 1;
+            }
+            std::stringstream out;
+            out << "(";
+            build_min_max_int(min_value, max_value, out);
+            out << ")";
+            return _add_rule(rule_name, out.str());
+        }
+        if (schema.empty() || schema_type == "object") {
+            return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
+        }
+        if (schema_type.is_null() && schema.is_object()) {
+            // No type constraint and no recognized structural keywords (e.g. {"description": "..."}).
+            // Per JSON Schema semantics this is equivalent to {} and accepts any value.
+            return _add_rule(rule_name, _add_primitive("value", PRIMITIVE_RULES.at("value")));
+        }
+        if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
+            _errors.push_back("Unrecognized schema: " + schema.dump());
+            return "";
+        }
+        // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
+        return _add_primitive(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>()));
     }
 
     void check_errors() {
@@ -989,6 +1102,134 @@ public:
     }
 };
 
+// common_schema_info implementation (pimpl)
+
+common_schema_info::common_schema_info()
+    : impl_(std::make_unique<common_schema_converter>(
+        [](const std::string &) { return json(); },
+        false)) {}
+
+common_schema_info::~common_schema_info() = default;
+
+common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
+common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
+
+void common_schema_info::resolve_refs(common_json & schema) {
+    impl_->resolve_refs(schema, "");
+}
+
+// Determines if a JSON schema can resolve to a string type through any path.
+// Some models emit raw string values rather than JSON-encoded strings for string parameters.
+// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
+// true, allowing callers to handle the value as a raw string for simplicity.
+bool common_schema_info::resolves_to_string(const common_json & schema) {
+    std::unordered_set<std::string> visited_refs;
+
+    std::function<bool(const json &)> check = [&](const json & s) -> bool {
+        if (!s.is_object()) {
+            return false;
+        }
+
+        // Handle $ref
+        if (s.contains("$ref")) {
+            const std::string & ref = s["$ref"];
+            if (visited_refs.find(ref) != visited_refs.end()) {
+                // Circular reference, assume not a string to be safe
+                return false;
+            }
+            visited_refs.insert(ref);
+            auto it = impl_->_refs.find(ref);
+            if (it != impl_->_refs.end()) {
+                return check(it->second);
+            }
+            return false;
+        }
+
+        // Check type field
+        if (s.contains("type")) {
+            const json & schema_type = s["type"];
+            if (schema_type.is_string()) {
+                if (schema_type == "string") {
+                    return true;
+                }
+            } else if (schema_type.is_array()) {
+                // Type can be an array like ["string", "null"]
+                for (const auto & t : schema_type) {
+                    if (t == "string") {
+                        return true;
+                    }
+                }
+            }
+        }
+
+        // Check oneOf/anyOf - if any alternative can be a string
+        if (s.contains("oneOf")) {
+            for (const auto & alt : s["oneOf"]) {
+                if (check(alt)) {
+                    return true;
+                }
+            }
+        }
+        if (s.contains("anyOf")) {
+            for (const auto & alt : s["anyOf"]) {
+                if (check(alt)) {
+                    return true;
+                }
+            }
+        }
+
+        // Check allOf - all components must be compatible with string type
+        if (s.contains("allOf")) {
+            bool all_string = true;
+            for (const auto & component : s["allOf"]) {
+                if (!check(component)) {
+                    all_string = false;
+                    break;
+                }
+            }
+            if (all_string) {
+                return true;
+            }
+        }
+
+        // Check const - if the constant value is a string
+        if (s.contains("const")) {
+            if (s["const"].is_string()) {
+                return true;
+            }
+        }
+
+        // Check enum - if any enum value is a string
+        if (s.contains("enum")) {
+            for (const auto & val : s["enum"]) {
+                if (val.is_string()) {
+                    return true;
+                }
+            }
+        }
+
+        // String-specific keywords imply string type
+        if (s.contains("pattern") || s.contains("minLength") || s.contains("maxLength")) {
+            return true;
+        }
+
+        // Check format - many formats imply string
+        if (s.contains("format")) {
+            const std::string & fmt = s["format"];
+            if (fmt == "date" || fmt == "time" || fmt == "date-time" ||
+                fmt == "uri" || fmt == "email" || fmt == "hostname" ||
+                fmt == "ipv4" || fmt == "ipv6" || fmt == "uuid" ||
+                fmt.find("uuid") == 0) {
+                return true;
+            }
+        }
+
+        return false;
+    };
+
+    return check(schema);
+}
+
 std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) {
 #ifdef LLAMA_USE_LLGUIDANCE
     if (!force_gbnf) {
@@ -997,29 +1238,25 @@ std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf)
 #else
     (void)force_gbnf;
 #endif // LLAMA_USE_LLGUIDANCE
-    try {
-        return json_schema_to_grammar(common_chat_schema_from_json(schema));
-    } catch (const std::runtime_error & e) {
-        throw std::invalid_argument(std::string("JSON schema conversion failed:\n") + e.what());
-    }
-}
-
-std::string json_schema_to_grammar(const common_chat_schema_document & schema) {
-    common_chat_schema_converter converter(false);
-    converter.visit(*schema.root, "");
-    converter.check_errors();
-    return converter.format_grammar();
+    return build_grammar([&](const common_grammar_builder & callbacks) {
+        auto copy = schema;
+        callbacks.resolve_refs(copy);
+        callbacks.add_schema("", copy);
+    });
 }
 
 std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options) {
-    common_chat_schema_converter converter(options.dotall);
+    common_schema_converter converter([&](const std::string &) { return json(); }, options.dotall);
     common_grammar_builder builder {
         /* .add_rule = */ [&](const std::string & name, const std::string & rule) {
             return converter._add_rule(name, rule);
         },
-        /* .add_schema = */ [&](const std::string & name, const common_chat_schema & schema) {
-            return converter.add_schema(name == "root" ? "" : name, schema);
+        /* .add_schema = */ [&](const std::string & name, const common_json & schema) {
+            return converter.visit(schema, name == "root" ? "" : name);
         },
+        /* .resolve_refs = */ [&](common_json & schema) {
+            converter.resolve_refs(schema, "");
+        }
     };
     cb(builder);
     converter.check_errors();

common/json-schema-to-grammar.h

@@ -1,17 +1,37 @@
 #pragma once
 
-#include "json-schema.h"
 #include "json.h"
 
 #include <functional>
+#include <memory>
 #include <string>
 
-std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf = false);
-std::string json_schema_to_grammar(const common_chat_schema_document & schema);
+std::string json_schema_to_grammar(const common_json & schema,
+                                   bool force_gbnf = false);
+
+class common_schema_converter;
+
+// Probes a JSON schema to extract information about its structure and type constraints.
+class common_schema_info {
+    std::unique_ptr<common_schema_converter> impl_;
+
+  public:
+    common_schema_info();
+    ~common_schema_info();
+
+    common_schema_info(const common_schema_info &) = delete;
+    common_schema_info & operator=(const common_schema_info &) = delete;
+    common_schema_info(common_schema_info &&) noexcept;
+    common_schema_info & operator=(common_schema_info &&) noexcept;
+
+    void resolve_refs(common_json & schema);
+    bool resolves_to_string(const common_json & schema);
+};
 
 struct common_grammar_builder {
-    std::function<std::string(const std::string &, const std::string &)>    add_rule;
-    std::function<std::string(const std::string &, const common_chat_schema &)> add_schema;
+    std::function<std::string(const std::string &, const std::string &)> add_rule;
+    std::function<std::string(const std::string &, const common_json &)> add_schema;
+    std::function<void(common_json &)> resolve_refs;
 };
 
 struct common_grammar_options {

common/json-schema.cpp

@@ -1,514 +0,0 @@
-#include "json-schema.h"
-#include "common.h"
-
-#include <cmath>
-#include <map>
-#include <stdexcept>
-#include <string>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-class common_chat_schema_builder {
-    const common_json &      root_;
-    common_chat_schema_document & doc_;
-
-    // the targets built here, moved into doc_ once the whole schema is built
-    std::map<std::string, common_chat_schema_ptr> refs_;
-
-    // ref nodes get their target once every $ref is built, a cycle would otherwise need it too early
-    std::vector<common_chat_schema_ref *> pending_;
-
-    [[noreturn]] static void fail(const std::string & path, const std::string & msg) {
-        throw std::runtime_error("JSON schema error at " + path + ": " + msg);
-    }
-
-    static int get_count(const common_json & schema, const std::string & key, const std::string & path, int def) {
-        if (!schema.contains(key)) {
-            return def;
-        }
-        const common_json & value = schema.at(key);
-        if (!value.is_number_integer() || value.get<int>() < 0) {
-            fail(path, key + " must be a non-negative integer");
-        }
-        return value.get<int>();
-    }
-
-    // a fractional bound is rounded inwards, towards the integers it still admits
-    static int64_t get_bound(const common_json & schema, const std::string & key, const std::string & path, bool round_up) {
-        const common_json & value = schema.at(key);
-        if (value.is_number_integer()) {
-            return value.get<int64_t>();
-        }
-        if (!value.is_number()) {
-            fail(path, key + " must be a number");
-        }
-        double d = value.get<double>();
-        return (int64_t) (round_up ? std::ceil(d) : std::floor(d));
-    }
-
-    static common_chat_schema::string_format get_format(const common_json & schema, const std::string & path) {
-        if (!schema.contains("format")) {
-            return common_chat_schema::FORMAT_NONE;
-        }
-        const common_json & value = schema.at("format");
-        if (!value.is_string()) {
-            fail(path, "format must be a string");
-        }
-        std::string format = value.get<std::string>();
-        if (format == "date") {
-            return common_chat_schema::FORMAT_DATE;
-        }
-        if (format == "time") {
-            return common_chat_schema::FORMAT_TIME;
-        }
-        if (format == "date-time") {
-            return common_chat_schema::FORMAT_DATE_TIME;
-        }
-        if (format == "uuid" || (format.size() == 5 && format.compare(0, 4, "uuid") == 0 && format[4] >= '1' && format[4] <= '5')) {
-            return common_chat_schema::FORMAT_UUID;
-        }
-        return common_chat_schema::FORMAT_NONE;
-    }
-
-    const common_json & resolve_ref(const std::string & ref, const std::string & path) {
-        const common_json * target = &root_;
-        auto tokens = string_split(ref.substr(1), "/");
-        for (size_t i = 1; i < tokens.size(); i++) {
-            const std::string & sel = tokens[i];
-            if (target->is_object() && target->contains(sel)) {
-                target = &target->at(sel);
-            } else if (target->is_array()) {
-                size_t idx;
-                try {
-                    idx = std::stoull(sel);
-                } catch (const std::logic_error &) {
-                    idx = target->size();
-                }
-                if (idx >= target->size()) {
-                    fail(path, "cannot resolve $ref " + ref + ", " + sel + " is out of range");
-                }
-                target = &target->at(idx);
-            } else {
-                fail(path, "cannot resolve $ref " + ref + ", " + sel + " not found");
-            }
-        }
-        return *target;
-    }
-
-    common_chat_schema_ptr build_ref(const common_json & value, const std::string & path) {
-        if (!value.is_string()) {
-            fail(path, "$ref must be a string");
-        }
-        std::string ref = value.get<std::string>();
-        if (ref.compare(0, 2, "#/") != 0) {
-            fail(path, "unsupported $ref " + ref + ", only references into the same document are supported");
-        }
-        if (refs_.find(ref) == refs_.end()) {
-            // reserve the key first, so that a cycle back to this $ref stops here
-            refs_[ref] = nullptr;
-            refs_[ref] = build_node(resolve_ref(ref, path), ref);
-        }
-        auto node = std::make_unique<common_chat_schema_ref>(ref);
-        pending_.push_back(node.get());
-        return node;
-    }
-
-    template <typename T>
-    common_chat_schema_ptr build_alternatives(const common_json & alts, const std::string & path) {
-        if (!alts.is_array()) {
-            fail(path, "must be an array of schemas");
-        }
-        if (alts.empty()) {
-            fail(path, "must not be empty");
-        }
-        auto node = std::make_unique<T>();
-        size_t i = 0;
-        for (const auto & alt : alts) {
-            node->children.push_back(build_node(alt, path + "/" + std::to_string(i++)));
-        }
-        return node;
-    }
-
-    common_chat_schema_ptr build_object(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_object>();
-
-        std::unordered_set<std::string> required;
-        if (schema.contains("required") && schema.at("required").is_array()) {
-            for (const auto & name : schema.at("required")) {
-                if (name.is_string()) {
-                    required.insert(name.get<std::string>());
-                }
-            }
-        }
-
-        if (schema.contains("properties")) {
-            const common_json & properties = schema.at("properties");
-            if (!properties.is_object()) {
-                fail(path, "properties must be an object");
-            }
-            for (const auto & [name, prop] : properties.items()) {
-                node->properties.push_back({name, build_node(prop, path + "/properties/" + name), required.count(name) > 0});
-            }
-        }
-
-        if (schema.contains("additionalProperties")) {
-            const common_json & additional = schema.at("additionalProperties");
-            if (additional.is_boolean()) {
-                if (additional.get<bool>()) {
-                    node->additional_properties = std::make_unique<common_chat_schema_any>();
-                }
-            } else if (additional.is_object()) {
-                node->additional_properties = build_node(additional, path + "/additionalProperties");
-            } else {
-                fail(path, "additionalProperties must be a boolean or a schema");
-            }
-        } else if (!schema.contains("properties")) {
-            // {"type": "object"} on its own accepts any object
-            node->additional_properties = std::make_unique<common_chat_schema_any>();
-        }
-
-        return node;
-    }
-
-    common_chat_schema_ptr build_array(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_array>();
-        if (schema.contains("items") || schema.contains("prefixItems")) {
-            // "items" wins when both are present; as in the converter, a schema instead of an array is the item schema
-            const std::string key = schema.contains("items") ? "items" : "prefixItems";
-            const common_json & items = schema.at(key);
-            if (items.is_array()) {
-                auto tuple = std::make_unique<common_chat_schema_tuple>();
-                size_t i = 0;
-                for (const auto & item : items) {
-                    tuple->items.push_back(build_node(item, path + "/" + key + "/" + std::to_string(i++)));
-                }
-                return tuple;
-            }
-            node->items = build_node(items, path + "/" + key);
-        } else {
-            node->items = std::make_unique<common_chat_schema_any>();
-        }
-        node->min_items = get_count(schema, "minItems", path, 0);
-        node->max_items = get_count(schema, "maxItems", path, -1);
-        return node;
-    }
-
-    common_chat_schema_ptr build_string(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_string>();
-        if (schema.contains("pattern")) {
-            const common_json & pattern = schema.at("pattern");
-            if (!pattern.is_string()) {
-                fail(path, "pattern must be a string");
-            }
-            node->pattern = pattern.get<std::string>();
-        }
-        node->format     = get_format(schema, path);
-        node->min_length = get_count(schema, "minLength", path, 0);
-        node->max_length = get_count(schema, "maxLength", path, -1);
-        return node;
-    }
-
-    common_chat_schema_ptr build_integer(const common_json & schema, const std::string & path) {
-        auto node = std::make_unique<common_chat_schema_integer>();
-        if (schema.contains("minimum")) {
-            node->minimum = get_bound(schema, "minimum", path, /* round_up */ true);
-        } else if (schema.contains("exclusiveMinimum")) {
-            node->minimum = get_bound(schema, "exclusiveMinimum", path, /* round_up */ false) + 1;
-        }
-        if (schema.contains("maximum")) {
-            node->maximum = get_bound(schema, "maximum", path, /* round_up */ false);
-        } else if (schema.contains("exclusiveMaximum")) {
-            node->maximum = get_bound(schema, "exclusiveMaximum", path, /* round_up */ true) - 1;
-        }
-        return node;
-    }
-
-    common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) {
-        if (!schema.is_object()) {
-            fail(path, "schema must be an object");
-        }
-        if (schema.contains("$ref")) {
-            return build_ref(schema.at("$ref"), path);
-        }
-        if (schema.contains("oneOf") || schema.contains("anyOf")) {
-            const std::string key = schema.contains("oneOf") ? "oneOf" : "anyOf";
-            return build_alternatives<common_chat_schema_any_of>(schema.at(key), path + "/" + key);
-        }
-
-        common_json type;
-        if (schema.contains("type")) {
-            type = schema.at("type");
-        }
-        if (type.is_array()) {
-            // {"type": ["a", "b"], ...} is {"anyOf": [{"type": "a", ...}, {"type": "b", ...}]}
-            if (type.empty()) {
-                fail(path, "type must not be empty");
-            }
-            auto node = std::make_unique<common_chat_schema_any_of>();
-            size_t i = 0;
-            for (const auto & t : type) {
-                common_json alt = schema;
-                alt["type"] = t;
-                node->children.push_back(build_node(alt, path + "/type/" + std::to_string(i++)));
-            }
-            return node;
-        }
-        if (schema.contains("const")) {
-            return std::make_unique<common_chat_schema_const>(schema.at("const"));
-        }
-        if (schema.contains("enum")) {
-            const common_json & values = schema.at("enum");
-            if (!values.is_array() || values.empty()) {
-                fail(path, "enum must be a non-empty array");
-            }
-            auto node = std::make_unique<common_chat_schema_enum>();
-            for (const auto & value : values) {
-                node->values.push_back(value);
-            }
-            return node;
-        }
-        if (!type.is_null() && !type.is_string()) {
-            fail(path, "type must be a string or an array of strings");
-        }
-
-        const std::string type_name = type.is_string() ? type.get<std::string>() : "";
-        const bool has_properties = schema.contains("properties") ||
-            (schema.contains("additionalProperties") && schema.at("additionalProperties") != true);
-
-        if (type_name.empty()) {
-            // without a type the structural keywords decide, in the same order as the converter
-            if (has_properties) {
-                return build_object(schema, path);
-            }
-            if (schema.contains("allOf")) {
-                return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
-            }
-            if (schema.contains("items") || schema.contains("prefixItems")) {
-                return build_array(schema, path);
-            }
-            if (schema.contains("pattern") || schema.contains("minLength") || schema.contains("maxLength") || get_format(schema, path) != common_chat_schema::FORMAT_NONE) {
-                return build_string(schema, path);
-            }
-            return std::make_unique<common_chat_schema_any>();
-        }
-        if (type_name == "object") {
-            if (!has_properties && schema.contains("allOf")) {
-                return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
-            }
-            return build_object(schema, path);
-        }
-        if (type_name == "string") {
-            if (schema.contains("allOf")) {
-                return build_alternatives<common_chat_schema_all_of>(schema.at("allOf"), path + "/allOf");
-            }
-            return build_string(schema, path);
-        }
-        if (type_name == "array") {
-            return build_array(schema, path);
-        }
-        if (type_name == "integer") {
-            return build_integer(schema, path);
-        }
-        if (type_name == "number") {
-            return std::make_unique<common_chat_schema_number>();
-        }
-        if (type_name == "boolean") {
-            return std::make_unique<common_chat_schema_boolean>();
-        }
-        if (type_name == "null") {
-            return std::make_unique<common_chat_schema_null>();
-        }
-        fail(path, "unrecognized type " + type_name);
-    }
-
-  public:
-    common_chat_schema_builder(const common_json & root, common_chat_schema_document & doc) : root_(root), doc_(doc) {}
-
-    common_chat_schema_ptr build() {
-        auto node = build_node(root_, "#");
-        for (auto & entry : refs_) {
-            doc_.refs[entry.first] = std::move(entry.second);
-        }
-        for (auto * ref : pending_) {
-            ref->target = doc_.refs.at(ref->ref).get();
-        }
-        return node;
-    }
-};
-
-common_chat_schema_document common_chat_schema_from_json(const common_json & schema) {
-    common_chat_schema_document doc;
-    doc.root = common_chat_schema_builder(schema, doc).build();
-    return doc;
-}
-
-static common_chat_schema::value_type json_type(const common_json & value) {
-    if (value.is_null()) {
-        return common_chat_schema::TYPE_NULL;
-    }
-    if (value.is_boolean()) {
-        return common_chat_schema::TYPE_BOOLEAN;
-    }
-    if (value.is_number_integer()) {
-        return common_chat_schema::TYPE_INTEGER;
-    }
-    if (value.is_number()) {
-        return common_chat_schema::TYPE_NUMBER;
-    }
-    if (value.is_string()) {
-        return common_chat_schema::TYPE_STRING;
-    }
-    if (value.is_array()) {
-        return common_chat_schema::TYPE_ARRAY;
-    }
-    return common_chat_schema::TYPE_OBJECT;
-}
-
-static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
-    switch (s.kind()) {
-        case common_chat_schema::KIND_ANY:
-            return common_chat_schema::type_set::all();
-        case common_chat_schema::KIND_NULL:
-            return { common_chat_schema::TYPE_NULL };
-        case common_chat_schema::KIND_BOOLEAN:
-            return { common_chat_schema::TYPE_BOOLEAN };
-        case common_chat_schema::KIND_NUMBER:
-            return { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER };
-        case common_chat_schema::KIND_INTEGER:
-            return { common_chat_schema::TYPE_INTEGER };
-        case common_chat_schema::KIND_STRING:
-            return { common_chat_schema::TYPE_STRING };
-        case common_chat_schema::KIND_ARRAY:
-        case common_chat_schema::KIND_TUPLE:
-            return { common_chat_schema::TYPE_ARRAY };
-        case common_chat_schema::KIND_OBJECT:
-            return { common_chat_schema::TYPE_OBJECT };
-        case common_chat_schema::KIND_CONST:
-            return { json_type(static_cast<const common_chat_schema_const &>(s).value) };
-        case common_chat_schema::KIND_ENUM: {
-            common_chat_schema::type_set types;
-            for (const auto & value : static_cast<const common_chat_schema_enum &>(s).values) {
-                types.add(json_type(value));
-            }
-            return types;
-        }
-        case common_chat_schema::KIND_REF: {
-            const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
-            if (!target || !visited.insert(target).second) {
-                // a cycle contributes no type, to be safe
-                return {};
-            }
-            auto types = value_types_impl(*target, visited);
-            visited.erase(target);
-            return types;
-        }
-        case common_chat_schema::KIND_ANY_OF: {
-            common_chat_schema::type_set types;
-            for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                types |= value_types_impl(*child, visited);
-            }
-            return types;
-        }
-        case common_chat_schema::KIND_ALL_OF: {
-            auto types = common_chat_schema::type_set::all();
-            for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
-                types &= value_types_impl(*child, visited);
-            }
-            return types;
-        }
-    }
-    return {};
-}
-
-common_chat_schema::type_set common_chat_schema::value_types() const {
-    std::unordered_set<const common_chat_schema *> visited;
-    return value_types_impl(*this, visited);
-}
-
-static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
-    switch (s.kind()) {
-        case common_chat_schema::KIND_STRING:
-            return true;
-        case common_chat_schema::KIND_CONST:
-            return static_cast<const common_chat_schema_const &>(s).value.is_string();
-        case common_chat_schema::KIND_ENUM:
-            for (const auto & v : static_cast<const common_chat_schema_enum &>(s).values) {
-                if (v.is_string()) {
-                    return true;
-                }
-            }
-            return false;
-        case common_chat_schema::KIND_REF: {
-            // a cycle is taken as not a string, to be safe
-            const auto * target = static_cast<const common_chat_schema_ref &>(s).target;
-            if (!target || !visited.insert(target).second) {
-                return false;
-            }
-            bool result = may_be_string_impl(*target, visited);
-            visited.erase(target);
-            return result;
-        }
-        case common_chat_schema::KIND_ANY_OF:
-            for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                if (may_be_string_impl(*child, visited)) {
-                    return true;
-                }
-            }
-            return false;
-        case common_chat_schema::KIND_ALL_OF: {
-            // every child must allow a string, an any child constrains nothing
-            bool any_string = false;
-            for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
-                if (child->kind() == common_chat_schema::KIND_ANY) {
-                    continue;
-                }
-                if (!may_be_string_impl(*child, visited)) {
-                    return false;
-                }
-                any_string = true;
-            }
-            return any_string;
-        }
-        default:
-            return false;
-    }
-}
-
-bool common_chat_schema::may_be_string() const {
-    std::unordered_set<const common_chat_schema *> visited;
-    return may_be_string_impl(*this, visited);
-}
-
-const char * common_chat_schema::kind_name(node_kind kind) {
-    switch (kind) {
-        case KIND_ANY:     return "any";
-        case KIND_REF:     return "ref";
-        case KIND_ANY_OF:  return "anyOf";
-        case KIND_ALL_OF:  return "allOf";
-        case KIND_CONST:   return "const";
-        case KIND_ENUM:    return "enum";
-        case KIND_NULL:    return "null";
-        case KIND_BOOLEAN: return "boolean";
-        case KIND_NUMBER:  return "number";
-        case KIND_INTEGER: return "integer";
-        case KIND_STRING:  return "string";
-        case KIND_ARRAY:   return "array";
-        case KIND_TUPLE:   return "tuple";
-        case KIND_OBJECT:  return "object";
-    }
-    return "?";
-}
-
-const char * common_chat_schema::type_name(value_type type) {
-    switch (type) {
-        case TYPE_NULL:    return "null";
-        case TYPE_BOOLEAN: return "boolean";
-        case TYPE_NUMBER:  return "number";
-        case TYPE_INTEGER: return "integer";
-        case TYPE_STRING:  return "string";
-        case TYPE_ARRAY:   return "array";
-        case TYPE_OBJECT:  return "object";
-    }
-    return "?";
-}

common/json-schema.h

@@ -1,198 +0,0 @@
-#pragma once
-
-#include "json.h"
-
-#include <cstdint>
-#include <initializer_list>
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-// JSON schema, covering the subset that json_schema_to_grammar() can convert.
-
-struct common_chat_schema {
-    enum node_kind {
-        KIND_ANY,
-        KIND_REF,
-        KIND_ANY_OF,
-        KIND_ALL_OF,
-        KIND_CONST,
-        KIND_ENUM,
-        KIND_NULL,
-        KIND_BOOLEAN,
-        KIND_NUMBER,
-        KIND_INTEGER,
-        KIND_STRING,
-        KIND_ARRAY,
-        KIND_TUPLE,
-        KIND_OBJECT,
-    };
-
-    enum value_type {
-        TYPE_NULL,
-        TYPE_BOOLEAN,
-        TYPE_NUMBER,
-        TYPE_INTEGER,
-        TYPE_STRING,
-        TYPE_ARRAY,
-        TYPE_OBJECT,
-    };
-
-    enum string_format {
-        FORMAT_NONE,
-        FORMAT_UUID,  // uuid, uuid1 .. uuid5
-        FORMAT_DATE,
-        FORMAT_TIME,
-        FORMAT_DATE_TIME,
-    };
-
-    class type_set {
-        uint32_t mask_ = 0;
-
-      public:
-        type_set() = default;
-        type_set(std::initializer_list<value_type> types) {
-            for (auto type : types) {
-                add(type);
-            }
-        }
-
-        static type_set all() {
-            return { TYPE_NULL, TYPE_BOOLEAN, TYPE_NUMBER, TYPE_INTEGER, TYPE_STRING, TYPE_ARRAY, TYPE_OBJECT };
-        }
-
-        void add(value_type type) { mask_ |= 1u << type; }
-
-        bool has(value_type type) const { return (mask_ & (1u << type)) != 0; }
-        bool is_only(value_type type) const { return mask_ == (1u << type); }
-        bool empty() const { return mask_ == 0; }
-
-        type_set & operator|=(const type_set & other) { mask_ |= other.mask_; return *this; }
-        type_set & operator&=(const type_set & other) { mask_ &= other.mask_; return *this; }
-
-        bool operator==(const type_set & other) const { return mask_ == other.mask_; }
-        bool operator!=(const type_set & other) const { return mask_ != other.mask_; }
-    };
-
-    virtual ~common_chat_schema() = default;
-    virtual node_kind kind() const = 0;
-
-    type_set value_types() const;
-
-    // Whether a value matching the schema may be a string, through any branch of it.
-    bool may_be_string() const;
-
-    static const char * kind_name(node_kind kind);
-    static const char * type_name(value_type type);
-};
-
-using common_chat_schema_ptr = std::unique_ptr<common_chat_schema>;
-
-struct common_chat_schema_any : common_chat_schema {
-    node_kind kind() const override { return KIND_ANY; }
-};
-
-// {"$ref": "#/..."}, only references into the same document are supported
-struct common_chat_schema_ref : common_chat_schema {
-    std::string                ref;
-    const common_chat_schema * target = nullptr;  // owned by common_chat_schema_document::refs
-
-    explicit common_chat_schema_ref(std::string ref) : ref(std::move(ref)) {}
-
-    node_kind kind() const override { return KIND_REF; }
-};
-
-// oneOf / anyOf, or a "type" array expanded to one alternative per type
-struct common_chat_schema_any_of : common_chat_schema {
-    std::vector<common_chat_schema_ptr> children;
-
-    node_kind kind() const override { return KIND_ANY_OF; }
-};
-
-struct common_chat_schema_all_of : common_chat_schema {
-    std::vector<common_chat_schema_ptr> children;
-
-    node_kind kind() const override { return KIND_ALL_OF; }
-};
-
-struct common_chat_schema_const : common_chat_schema {
-    common_json value;
-
-    explicit common_chat_schema_const(common_json value) : value(std::move(value)) {}
-
-    node_kind kind() const override { return KIND_CONST; }
-};
-
-struct common_chat_schema_enum : common_chat_schema {
-    std::vector<common_json> values;
-
-    node_kind kind() const override { return KIND_ENUM; }
-};
-
-struct common_chat_schema_null : common_chat_schema {
-    node_kind kind() const override { return KIND_NULL; }
-};
-
-struct common_chat_schema_boolean : common_chat_schema {
-    node_kind kind() const override { return KIND_BOOLEAN; }
-};
-
-struct common_chat_schema_number : common_chat_schema {
-    node_kind kind() const override { return KIND_NUMBER; }
-};
-
-// bounds are inclusive, exclusiveMinimum / exclusiveMaximum are folded in
-struct common_chat_schema_integer : common_chat_schema {
-    int64_t minimum = INT64_MIN;  // INT64_MIN for unbounded
-    int64_t maximum = INT64_MAX;  // INT64_MAX for unbounded
-
-    node_kind kind() const override { return KIND_INTEGER; }
-};
-
-struct common_chat_schema_string : common_chat_schema {
-    std::string   pattern;  // empty when absent
-    string_format format     = FORMAT_NONE;
-    int           min_length = 0;
-    int           max_length = -1;  // -1 for unbounded
-
-    node_kind kind() const override { return KIND_STRING; }
-};
-
-struct common_chat_schema_array : common_chat_schema {
-    common_chat_schema_ptr items;  // a common_chat_schema_any when "items" is absent
-    int                    min_items = 0;
-    int                    max_items = -1;  // -1 for unbounded
-
-    node_kind kind() const override { return KIND_ARRAY; }
-};
-
-struct common_chat_schema_tuple : common_chat_schema {
-    std::vector<common_chat_schema_ptr> items;
-
-    node_kind kind() const override { return KIND_TUPLE; }
-};
-
-struct common_chat_schema_property {
-    std::string            name;
-    common_chat_schema_ptr schema;
-    bool                   required = false;
-};
-
-struct common_chat_schema_object : common_chat_schema {
-    std::vector<common_chat_schema_property> properties;             // in schema order
-    common_chat_schema_ptr                   additional_properties;  // null when not allowed
-
-    node_kind kind() const override { return KIND_OBJECT; }
-};
-
-struct common_chat_schema_document {
-    common_chat_schema_ptr                        root;
-    std::map<std::string, common_chat_schema_ptr> refs;
-};
-
-// A document shared by the PEG parsers built from its nodes, which it keeps alive
-using common_chat_schema_document_ptr = std::shared_ptr<const common_chat_schema_document>;
-
-// Throws std::runtime_error when the schema falls outside the supported subset.
-common_chat_schema_document common_chat_schema_from_json(const common_json & schema);

common/json.cpp

@@ -78,21 +78,19 @@ common_json_value::common_json_value(const common_json & val) :
 common_json_value::common_json_value(common_json && val) :
     type(VAL_JSON), val_json(std::make_shared<common_json>(std::move(val))) {}
 
-// the ctors and get<T>() below are explicit specializations, giving strong symbols
-// an explicit instantiation is a weak symbol, dropped by some LTO builds (clang-cl)
 template <typename T>
-static std::shared_ptr<common_json> set_json(const std::set<T> & vals) {
+common_json_value::common_json_value(const std::set<T> & vals) : type(VAL_JSON) {
     common_json out = common_json::array();
 
     for (const auto & val : vals) {
         out.push_back(val);
     }
 
-    return std::make_shared<common_json>(std::move(out));
+    val_json = std::make_shared<common_json>(std::move(out));
 }
 
 // a set value is usable only for the types below
-#define COMMON_JSON_SET(...) template <> common_json_value::common_json_value(const std::set<__VA_ARGS__> & vals) : type(VAL_JSON), val_json(set_json(vals)) {}
+#define COMMON_JSON_SET(...) template common_json_value::common_json_value(const std::set<__VA_ARGS__> &);
 
 COMMON_JSON_SET(int)
 COMMON_JSON_SET(std::string)
@@ -100,45 +98,56 @@ COMMON_JSON_SET(std::string)
 #undef COMMON_JSON_SET
 
 template <typename T>
-static std::shared_ptr<common_json> map_json(const T & vals) {
+common_json_value::common_json_value(const std::map<std::string, T> & vals) : type(VAL_JSON) {
     common_json out = common_json::object();
 
     for (const auto & val : vals) {
         out.set({ val.first, val.second });
     }
 
-    return std::make_shared<common_json>(std::move(out));
+    val_json = std::make_shared<common_json>(std::move(out));
 }
 
 // a map value is usable only for the types below
-#define COMMON_JSON_MAP(...) template <> common_json_value::common_json_value(const std::map<std::string, __VA_ARGS__> & vals) : type(VAL_JSON), val_json(map_json(vals)) {}
+#define COMMON_JSON_MAP(...) template common_json_value::common_json_value(const std::map<std::string, __VA_ARGS__> &);
 
 COMMON_JSON_MAP(bool)
 COMMON_JSON_MAP(std::string)
 
 #undef COMMON_JSON_MAP
 
+template <typename T>
+common_json_value::common_json_value(const std::unordered_map<std::string, T> & vals) : type(VAL_JSON) {
+    common_json out = common_json::object();
+
+    for (const auto & val : vals) {
+        out.set({ val.first, val.second });
+    }
+
+    val_json = std::make_shared<common_json>(std::move(out));
+}
+
 // an unordered map value is usable only for the types below
-#define COMMON_JSON_UMAP(...) template <> common_json_value::common_json_value(const std::unordered_map<std::string, __VA_ARGS__> & vals) : type(VAL_JSON), val_json(map_json(vals)) {}
+#define COMMON_JSON_UMAP(...) template common_json_value::common_json_value(const std::unordered_map<std::string, __VA_ARGS__> &);
 
 COMMON_JSON_UMAP(size_t)
 
 #undef COMMON_JSON_UMAP
 
 template <typename T>
-static std::shared_ptr<common_json> vec_json(const std::vector<T> & vals) {
+common_json_value::common_json_value(const std::vector<T> & vals) : type(VAL_JSON) {
     common_json out = common_json::array();
 
     for (const auto & val : vals) {
         out.push_back(val);
     }
 
-    return std::make_shared<common_json>(std::move(out));
+    val_json = std::make_shared<common_json>(std::move(out));
 }
 
 // a vector value is usable only for the types below
 // note: std::vector<bool> is not here, its proxy reference does not convert
-#define COMMON_JSON_VEC(...) template <> common_json_value::common_json_value(const std::vector<__VA_ARGS__> & vals) : type(VAL_JSON), val_json(vec_json(vals)) {}
+#define COMMON_JSON_VEC(...) template common_json_value::common_json_value(const std::vector<__VA_ARGS__> &);
 
 COMMON_JSON_VEC(int)
 COMMON_JSON_VEC(unsigned char)
@@ -395,6 +404,10 @@ common_json::items_view common_json::items() const {
     return items_view(const_cast<common_json *>(this), size());
 }
 
+template <typename T> T common_json::get() const {
+    return guard([&] { return as_json(this).get<T>(); });
+}
+
 // the backing library cannot build a common_json, so this one is just a copy
 template <> common_json common_json::get<common_json>() const {
     return *this;
@@ -402,7 +415,7 @@ template <> common_json common_json::get<common_json>() const {
 
 // get<T>() is usable only for the types below
 
-#define COMMON_JSON_GET(...) template <> __VA_ARGS__ common_json::get<__VA_ARGS__>() const { return guard([&] { return as_json(this).get<__VA_ARGS__>(); }); }
+#define COMMON_JSON_GET(...) template __VA_ARGS__ common_json::get<__VA_ARGS__>() const;
 
 COMMON_JSON_GET(bool)
 COMMON_JSON_GET(int)
@@ -422,12 +435,3 @@ COMMON_JSON_GET(std::vector<size_t>)
 COMMON_JSON_GET(std::unordered_map<std::string, size_t>)
 
 #undef COMMON_JSON_GET
-
-// must stay below the get<std::string> specialization
-common_json::operator std::string() const {
-    return get<std::string>();
-}
-
-std::string common_json::value(const std::string & key, const char * def) const {
-    return contains(key) ? at(key).get<std::string>() : std::string(def);
-}

common/json.h

@@ -221,14 +221,16 @@ class common_json {
     // implicit get<T>() for plain values, so they can be assigned to their C++ type directly
     // note: kept to this short list on purpose, a wider one makes j["key"] ambiguous
     // note: a numeric one would make "str = json;" ambiguous, a number converts to char too
-    operator std::string() const;
+    operator std::string() const { return get<std::string>(); }
 
     template <typename T>
     T value(const std::string & key, T def) const {
         return contains(key) ? at(key).get<T>() : def;
     }
 
-    std::string value(const std::string & key, const char * def) const;
+    std::string value(const std::string & key, const char * def) const {
+        return contains(key) ? at(key).get<std::string>() : std::string(def);
+    }
 
     // a JSON default needs no get<T>(), it is already the right type
     common_json value(const std::string & key, const common_json & def) const {

common/log.cpp

@@ -1,6 +1,5 @@
 #include "common.h"
 #include "log.h"
-#include "json.h"
 
 #include <chrono>
 #include <condition_variable>
@@ -37,16 +36,6 @@ void common_log_set_verbosity_thold(int verbosity) {
     common_log_verbosity_thold = verbosity;
 }
 
-static bool common_log_jsonl = false;
-
-bool common_log_get_jsonl(void) {
-    return common_log_jsonl;
-}
-
-void common_log_set_jsonl(bool jsonl) {
-    common_log_jsonl = jsonl;
-}
-
 static int64_t t_us() {
     return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 }
@@ -77,17 +66,6 @@ static const char* g_col[] = {
     "",
 };
 
-static const char * level_str(enum ggml_log_level level) {
-    switch (level) {
-        case GGML_LOG_LEVEL_DEBUG: return "debug";
-        case GGML_LOG_LEVEL_INFO:  return "info";
-        case GGML_LOG_LEVEL_WARN:  return "warn";
-        case GGML_LOG_LEVEL_ERROR: return "error";
-        case GGML_LOG_LEVEL_CONT:  return "cont";
-        default:                   return "none";
-    }
-}
-
 struct common_log_entry {
     enum ggml_log_level level {GGML_LOG_LEVEL_INFO};
 
@@ -96,8 +74,6 @@ struct common_log_entry {
     int64_t timestamp { 0 };
     bool is_end       { false }; // signals the worker thread to stop
     bool prefix       { false };
-    bool jsonl        { false };
-    bool is_json      { false }; // msg already holds a serialized JSON object
 
     common_log_entry(size_t size = 256) : msg(size) { }
 
@@ -112,29 +88,11 @@ struct common_log_entry {
 
             fcur = stdout;
 
-            if (level != GGML_LOG_LEVEL_NONE && !jsonl) {
+            if (level != GGML_LOG_LEVEL_NONE) {
                 fcur = stderr;
             }
         }
 
-        if (jsonl) {
-            if (is_json) {
-                fprintf(fcur, "%s\n", msg.data());
-                fflush(fcur);
-                return;
-            }
-
-            common_json obj = {
-                {"type",  "log"},
-                {"time",  timestamp},
-                {"level", level_str(level)},
-                {"msg",   msg.data()},
-            };
-            fprintf(fcur, "%s\n", obj.dump_safe().c_str());
-            fflush(fcur);
-            return;
-        }
-
         if (level != GGML_LOG_LEVEL_NONE && level != GGML_LOG_LEVEL_CONT && prefix) {
             if (timestamp) {
                 // [M.s.ms.us]
@@ -288,8 +246,6 @@ public:
         entry.is_end    = false;
         entry.level     = level;
         entry.prefix    = prefix;
-        entry.jsonl     = common_log_jsonl;
-        entry.is_json   = false;
         entry.timestamp = 0;
         if (timestamps) {
             entry.timestamp = t_us() - t_start;
@@ -299,42 +255,6 @@ public:
         cv_new.notify_one();
     }
 
-    void add_json(const char * type, const common_json & obj) {
-        const common_json full = {
-            {"type", type},
-            {"data", obj},
-        };
-
-        const std::string text = full.dump_safe();
-
-        std::unique_lock<std::mutex> lock(mtx);
-
-        // block if the queue is full
-        cv_full.wait(lock, [this]() { return !running || !is_full(); });
-
-        if (!running) {
-            // discard messages while the worker thread is paused
-            return;
-        }
-
-        auto & entry = queue[tail];
-
-        if (entry.msg.size() < text.size() + 1) {
-            entry.msg.resize(text.size() + 1);
-        }
-        memcpy(entry.msg.data(), text.c_str(), text.size() + 1);
-
-        entry.is_end    = false;
-        entry.level     = GGML_LOG_LEVEL_NONE;
-        entry.prefix    = false;
-        entry.jsonl     = true;
-        entry.is_json   = true;
-        entry.timestamp = 0;
-
-        tail = (tail + 1) % queue.size();
-        cv_new.notify_one();
-    }
-
     void resume() {
         std::lock_guard<std::mutex> lock(mtx);
 
@@ -486,14 +406,6 @@ void common_log_add(struct common_log * log, enum ggml_log_level level, const ch
     va_end(args);
 }
 
-void common_log_add_json(struct common_log * log, const char * type, const common_json & obj) {
-    if (!common_log_jsonl) {
-        return;
-    }
-
-    log->add_json(type, obj);
-}
-
 void common_log_set_file(struct common_log * log, const char * file) {
     log->set_file(file);
 }
@@ -526,7 +438,7 @@ void common_log_flush(struct common_log * log) {
     log->resume();
 }
 
-int common_log_get_verbosity(enum ggml_log_level level) {
+static int common_get_verbosity(enum ggml_log_level level) {
     switch (level) {
         case GGML_LOG_LEVEL_DEBUG: return LOG_LEVEL_DEBUG;
         case GGML_LOG_LEVEL_INFO:  return LOG_LEVEL_TRACE;
@@ -540,7 +452,7 @@ int common_log_get_verbosity(enum ggml_log_level level) {
 }
 
 void common_log_default_callback(enum ggml_log_level level, const char * text, void * /*user_data*/) {
-    auto verbosity = common_log_get_verbosity(level);
+    auto verbosity = common_get_verbosity(level);
     if (verbosity <= common_log_verbosity_thold) {
         common_log_add(common_log_main(), level, "%s", text);
     }

common/log.h

@@ -43,12 +43,6 @@ int  common_log_get_verbosity_thold(void);
 
 void common_log_set_verbosity_thold(int verbosity); // not thread-safe
 
-bool common_log_get_jsonl(void);
-
-void common_log_set_jsonl(bool jsonl); // not thread-safe
-
-int common_log_get_verbosity(enum ggml_log_level level);
-
 void common_log_default_callback(enum ggml_log_level level, const char * text, void * user_data);
 
 // the common_log uses an internal worker thread to print/write log messages
@@ -130,17 +124,3 @@ void common_log_flush         (struct common_log * log);                    // f
 #define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  verbosity, __VA_ARGS__)
 #define LOG_ERRV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, verbosity, __VA_ARGS__)
 #define LOG_CNTV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_CONT,  verbosity, __VA_ARGS__)
-
-class common_json; // defined in common/json.h
-
-// helper allows different types of json output
-// no-op if --log-jsonl is not set
-void common_log_add_json(struct common_log * log, const char * type, const common_json & data);
-
-// will only print if --log-jsonl is set
-#define LOG_JSON(type, data) \
-    do { \
-        if (common_log_get_jsonl()) { \
-            common_log_add_json(common_log_main(), type, data); \
-        } \
-    } while (0)

common/parsers/cohere2moe.cpp

@@ -1,141 +0,0 @@
-#include "parsers.h"
-
-// Cohere2 MoE (a.k.a. "North Code") parser.
-//
-// The assistant turn is fully marker-wrapped:
-//   <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
-//     <|START_THINKING|>{reasoning}<|END_THINKING|>
-//     then EITHER content:    <|START_TEXT|>{content}<|END_TEXT|>
-//          OR     tool calls: <|START_ACTION|>[
-//                                 {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...
-//                             ]<|END_ACTION|>
-//   <|END_OF_TURN_TOKEN|>
-//
-// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is
-// the template default), so the model's output continues from *inside* the thinking block. The
-// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix
-// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,
-// regardless of whether they came from the generation prompt or the generated text.
-common_chat_params common_chat_params_init_cohere2moe(const common_chat_template &          tmpl,
-                                                              const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    const std::string TURN_START    = "<|START_OF_TURN_TOKEN|>";
-    const std::string TURN_END      = "<|END_OF_TURN_TOKEN|>";
-    const std::string CHATBOT       = "<|CHATBOT_TOKEN|>";
-    const std::string USER          = "<|USER_TOKEN|>";
-    const std::string SYSTEM        = "<|SYSTEM_TOKEN|>";
-    const std::string THINK_START   = "<|START_THINKING|>";
-    const std::string THINK_END     = "<|END_THINKING|>";
-    const std::string TEXT_START    = "<|START_TEXT|>";
-    const std::string TEXT_END      = "<|END_TEXT|>";
-    const std::string ACTION_START  = "<|START_ACTION|>";
-    const std::string ACTION_END    = "<|END_ACTION|>";
-    const std::string RESULT_START  = "<|START_TOOL_RESULT|>";
-    const std::string RESULT_END    = "<|END_TOOL_RESULT|>";
-
-    // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.
-    const std::string GEN_PREFIX = TURN_START + CHATBOT;
-
-    data.prompt             = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt  = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking  = true;
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = {THINK_END};
-    data.preserved_tokens   = {
-        TURN_START, TURN_END, CHATBOT, USER, SYSTEM,
-        THINK_START, THINK_END,
-        TEXT_START, TEXT_END,
-        ACTION_START, ACTION_END,
-        RESULT_START, RESULT_END,
-    };
-
-    // Declare per-role message delimiters. Tool results are rendered with the
-    // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before
-    // the plain "system" one (it is a strict superset, and the role split tries delimiters in order).
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX },
-        { COMMON_CHAT_ROLE_USER,      TURN_START + USER },
-        { COMMON_CHAT_ROLE_TOOL,      TURN_START + SYSTEM + RESULT_START },
-        { COMMON_CHAT_ROLE_SYSTEM,    TURN_START + SYSTEM },
-    };
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += THINK_END + TEXT_START + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal(GEN_PREFIX);
-        auto end               = p.end();
-
-        // The thinking block is always present (the generation prompt forces <|START_THINKING|>).
-        // When extracting reasoning, capture its body; otherwise keep the whole block (markers
-        // included) inline as content, matching reasoning_format=NONE conventions.
-        common_peg_parser reasoning = p.eps();
-        if (extract_reasoning) {
-            reasoning = p.optional(p.literal(THINK_START) +
-                                   p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) +
-                                   p.optional(p.literal(THINK_END)));
-        } else {
-            reasoning = p.optional(p.content(p.literal(THINK_START) +
-                                             p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) +
-                                             p.optional(p.literal(THINK_END))));
-        }
-
-        auto text_content = has_response_format
-            ? p.literal(TEXT_START) +
-                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
-                p.optional(p.literal(TEXT_END))
-            : p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));
-
-        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-            return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;
-        }
-
-        auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-
-        // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>
-        auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,
-                                                /* force_tool_calls = */ true,
-                                                /* name_key         = */ "tool_name",
-                                                /* args_key         = */ "parameters",
-                                                /* array_wrapped    = */ true,
-                                                /* function_is_key  = */ false,
-                                                /* call_id_key      = */ "",
-                                                /* gen_call_id_key  = */ "tool_call_id",
-                                                /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });
-
-        // Content and tool calls are mutually exclusive in this format.
-        common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });
-
-        return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }
-        };
-    }
-
-    return data;
-}

common/parsers/deepseek.cpp

@@ -1,267 +0,0 @@
-#include "parsers.h"
-
-// The DeepSeek V4 reference implementation renders consecutive tool results into a single
-// user block, ordered by the tool call order of the preceding assistant message (matched
-// by tool call id) rather than by the order they appear in the conversation.
-static json deepseek_v4_sort_tool_results(const json & messages) {
-    json adjusted = messages;
-    std::map<std::string, size_t> call_order;
-
-    for (size_t i = 0; i < adjusted.size();) {
-        const auto & msg  = adjusted[i];
-        const auto   role = msg.value("role", "");
-
-        if (role == "assistant" && msg.contains("tool_calls") &&
-                msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
-            call_order.clear();
-            const auto & tool_calls = msg.at("tool_calls");
-            for (size_t idx = 0; idx < tool_calls.size(); idx++) {
-                auto id = tool_calls[idx].value("id", "");
-                if (!id.empty()) {
-                    call_order[id] = idx;
-                }
-            }
-            i++;
-            continue;
-        }
-
-        if (role != "user" && role != "tool") {
-            i++;
-            continue;
-        }
-
-        // collect a maximal run of user/tool messages - they render into one user block
-        std::vector<size_t> tool_positions;
-        size_t run_end = i;
-        for (; run_end < adjusted.size(); run_end++) {
-            const auto r = adjusted[run_end].value("role", "");
-            if (r == "tool") {
-                tool_positions.push_back(run_end);
-            } else if (r != "user") {
-                break;
-            }
-        }
-
-        if (tool_positions.size() > 1 && !call_order.empty()) {
-            std::vector<json> results;
-            results.reserve(tool_positions.size());
-            for (auto pos : tool_positions) {
-                results.push_back(adjusted[pos]);
-            }
-            std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {
-                const auto order = [&](const json & m) {
-                    auto it = call_order.find(m.value("tool_call_id", ""));
-                    return it == call_order.end() ? (size_t) 0 : it->second;
-                };
-                return order(a) < order(b);
-            });
-            for (size_t k = 0; k < tool_positions.size(); k++) {
-                adjusted[tool_positions[k]] = std::move(results[k]);
-            }
-        }
-
-        i = run_end;
-    }
-
-    return adjusted;
-}
-
-common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template &    tmpl,
-                                                                 const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    // V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"
-    // instead of "function_calls", renders tool results in tool call order and its
-    // non-thinking generation prompt ends with a bare </think> instead of an empty
-    // <think></think> pair.
-    const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
-
-    std::optional<json> adjusted_messages;
-    if (is_v4) {
-        adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
-    }
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    std::optional<json> additional_context;
-    if (is_v4 && has_response_format) {
-        additional_context = json{ { "response_format", inputs.json_schema } };
-    }
-
-    const std::string DSML         = "｜DSML｜";
-    const std::string THINK_START  = "<think>";
-    const std::string THINK_END    = "</think>";
-    const std::string TC_BLOCK     = is_v4 ? "tool_calls" : "function_calls";
-    const std::string FC_START     = "<" + DSML + TC_BLOCK + ">";
-    const std::string FC_END       = "</" + DSML + TC_BLOCK + ">";
-    const std::string INVOKE_START = "<" + DSML + "invoke";
-    const std::string INVOKE_END   = "</" + DSML + "invoke>";
-    const std::string PARAM_START  = "<" + DSML + "parameter";
-    const std::string PARAM_END    = "</" + DSML + "parameter>";
-    const std::string GEN_PROMPT   = "<｜Assistant｜>";
-    const std::string TC_SEPARATOR = "\n\n";
-
-    data.prompt = common_chat_template_direct_apply_impl(
-        tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(
-        tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
-    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking  = true;
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = {THINK_END, FC_START};
-    data.preserved_tokens   = {
-        DSML,
-        THINK_START,
-        THINK_END,
-    };
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        if (is_v4 && msg.reasoning_content.empty()) {
-            data.generation_prompt = GEN_PROMPT + THINK_END;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += msg.render_content();
-            }
-        } else {
-            data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += THINK_END + msg.render_content();
-            }
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    bool require_tools   = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-    bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal(GEN_PROMPT);
-        auto end               = p.end();
-
-        // build tool call section first since we might need it in reasoning
-        auto tool_choice = p.choice();
-        if (has_tool_calls) {
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                std::string  name     = function.at("name");
-
-                std::vector<common_peg_parser> required_parsers;
-                std::vector<common_peg_parser> optional_parsers;
-                foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
-                    bool is_string = param.schema->may_be_string();
-
-                    auto arg = p.tool_arg(
-                        p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param.name)) +
-                                        p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
-                        (is_string ?
-                             p.tool_arg_string_value(p.until(PARAM_END)) :
-                             p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param.name + "-schema",
-                                                            doc, *param.schema))) +
-                        p.tool_arg_close(p.literal(PARAM_END)));
-
-                    auto named_arg = p.rule("tool-" + name + "-arg-" + param.name, arg);
-                    if (param.required) {
-                        required_parsers.push_back(named_arg);
-                    } else {
-                        optional_parsers.push_back(named_arg);
-                    }
-                });
-
-                common_peg_parser args_seq = p.eps();
-                for (size_t i = 0; i < required_parsers.size(); i++) {
-                    if (i > 0) {
-                        args_seq = args_seq + p.space();
-                    }
-                    args_seq = args_seq + required_parsers[i];
-                }
-
-                if (!optional_parsers.empty()) {
-                    common_peg_parser any_opt = p.choice();
-                    for (const auto & opt : optional_parsers) {
-                        any_opt |= opt;
-                    }
-                    args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
-                }
-
-                common_peg_parser invoke_body = args_seq;
-                auto              func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") +
-                                                                   p.tool_name(p.literal(name)) + p.literal("\">\n")) +
-                                                       invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));
-
-                tool_choice |= p.rule("tool-" + name, func_parser);
-            });
-        }
-
-        common_peg_parser tool_calls = p.eps();
-        if (inputs.parallel_tool_calls) {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice +
-                p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
-        } else {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
-        }
-
-        auto reasoning = p.eps();
-        auto reasoning_with_tc = p.eps();
-        auto obligatory_tool_calls = tool_calls;
-        bool allow_reasoning_with_tc = false;
-
-        if (!require_tools) {
-            tool_calls = p.optional(tool_calls);
-        }
-
-        if (extract_reasoning && inputs.enable_thinking) {
-            reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
-            reasoning_with_tc = THINK_START +
-                p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
-                p.space() + obligatory_tool_calls;
-            allow_reasoning_with_tc = true;
-        } else if (extract_reasoning) {
-            // Thinking disabled but reasoning extraction requested: the generation prompt
-            // contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
-            // must still be consumed.
-            reasoning = is_v4
-                ? p.optional(p.literal(THINK_END))
-                : p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
-        }
-
-        if (has_response_format) {
-            auto response_format = p.rule("response-format",
-                p.literal("```json") + p.space() +
-                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
-                p.space() + p.literal("```"));
-            return generation_prompt + reasoning + response_format + end;
-        }
-
-        if (!has_tool_calls) {
-            return generation_prompt + reasoning + p.content(p.rest()) + end;
-        }
-
-        auto content_before_tools = p.negate(p.literal(THINK_START)) +
-            p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +
-            p.space();
-        return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :
-            generation_prompt + reasoning + content_before_tools + tool_calls + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = has_tools && !require_tools;
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
-        };
-    }
-
-    return data;
-}

common/parsers/functionary-v3-2.cpp

@@ -1,96 +0,0 @@
-#include "parsers.h"
-
-// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content}
-common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template &    tmpl,
-                                                                   const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.preserved_tokens  = {
-        ">>>all",
-    };
-
-    auto has_tools         = inputs.tools.is_array() && !inputs.tools.empty();
-    auto include_grammar   = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-        data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content();
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        // Functionary v3.2 format:
-        // - Normal content: >>>all\n{content}
-        // - Tool calls: >>>function_name\n{json_args}
-        // Generation prompt ends with ">>>" so model outputs recipient immediately
-
-        // Build content parser for >>>all\n{content}
-        // When tools are present, content stops before the next ">>>" (tool call)
-        // When no tools, content goes until end
-        auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>"));
-        auto content_until_end  = p.literal("all\n") + p.content(p.rest());
-        auto generation_prompt  = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>");
-
-        // If no tools or tool_choice is NONE, just parse content
-        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-            // When no tools, just match the prefix and capture everything after
-            return generation_prompt + content_until_end + p.end();
-        }
-
-        // Build tool call parsers for each available function
-        auto tool_choice = p.choice();
-        foreach_function(inputs.tools, [&](const json & tool) {
-            const auto & function = tool.at("function");
-            std::string  name     = function.at("name");
-            const auto   schema   = common_chat_tool_parameters(function);
-
-            // Tool format: >>>function_name\n{json_args}
-            auto tool_parser = p.tool(
-                p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) +
-                p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema))
-            );
-
-            tool_choice |= p.rule("tool-" + name, tool_parser);
-        });
-
-        auto content_only = content_until_end;
-        auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice));
-        auto content_and_tools = content_until_tool + tools_only;
-
-        auto ret = p.eps();
-        if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
-            if (inputs.parallel_tool_calls) {
-                ret = p.choice({ content_and_tools, tools_only }) + p.end();
-            } else {
-                ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end();
-            }
-        } else if (inputs.parallel_tool_calls) {
-            ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end();
-        } else {
-            auto content_and_tool = content_until_tool + tool_choice;
-            ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end();
-        }
-        return generation_prompt + ret;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-
-        data.grammar = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        // Grammar trigger for when the model starts outputting a tool call
-        // (after the initial ">>>" in the generation prompt but recipient other than "all")
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" }
-        };
-    }
-
-    return data;
-}

common/parsers/gemma4.cpp

@@ -1,303 +0,0 @@
-#include "parsers.h"
-
-namespace workaround {
-
-// Gemma4 uses a custom tool_responses field instead of role:tool messages.
-//
-// This will transform a sequence of messages:
-//   assistant(tool_call+) -> tool+ -> assistant(content)
-//
-// Into a single assistant message containing a tool_responses field:
-//   assistant(content + tool_call + tool_responses)
-//
-// This is necessary for the Gemma4 chat template to properly format the prompt.
-// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4
-struct gemma4_model_turn_builder {
-    json & messages;
-    size_t pos;
-    json tool_calls = json::array();
-    json tool_responses = json::array();
-    json content;
-    json reasoning_content;
-
-    gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}
-
-    void collect() {
-        // Collect the first assistant message
-        auto & msg = messages[pos];
-        if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
-            // According to the prompt formatting guide, we need to preserve reasoning_content
-            // between function calls. The current chat templates do not support this, but we will do it anyway.
-            reasoning_content = msg.at("reasoning_content");
-        }
-        for (auto & tc : msg.at("tool_calls")) {
-            tool_calls.push_back(tc);
-        }
-        pos++;
-
-        // Collect tool call results
-        while (pos < messages.size() && messages[pos].value("role", "") == "tool") {
-            collect_result(messages[pos]);
-            pos++;
-        }
-
-        // Check if the next assistant message is the final message
-        if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {
-            auto & next = messages[pos];
-            if (!has_tool_calls(next) && has_content(next)) {
-                content = next.at("content");
-                pos++;
-            }
-        }
-    }
-
-    void collect_result(const json & curr) {
-        json response;
-        if (curr.contains("content")) {
-            const auto & content = curr.at("content");
-            if (content.is_string()) {
-                // Try to parse the content as JSON; fall back to raw string
-                try {
-                    response = json::parse(content.get<std::string>());
-                } catch (...) {
-                    response = content;
-                }
-            } else {
-                response = content;
-            }
-        }
-
-        std::string name;
-
-        // Match name with corresponding tool call
-        size_t idx = tool_responses.size();
-        if (idx < tool_calls.size()) {
-            auto & tc = tool_calls[idx];
-            if (tc.contains("function")) {
-                name = tc.at("function").value("name", "");
-            }
-        }
-
-        // Fallback to the tool call id
-        if (name.empty()) {
-            name = curr.value("tool_call_id", "");
-        }
-
-        tool_responses.push_back({{"name", name}, {"response", response}});
-    }
-
-    json build() {
-        collect();
-
-        json msg = {
-            {"role", "assistant"},
-            {"tool_calls", tool_calls},
-        };
-        if (!tool_responses.empty()) {
-            msg["tool_responses"] = tool_responses;
-        }
-        if (!content.is_null()) {
-            msg["content"] = content;
-        }
-        if (!reasoning_content.is_null()) {
-            msg["reasoning_content"] = reasoning_content;
-        }
-        return msg;
-    }
-
-    static bool has_content(const json & msg) {
-        if (!msg.contains("content") || msg.at("content").is_null()) {
-            return false;
-        }
-        const auto & content = msg.at("content");
-        if (content.is_string() && !content.get<std::string>().empty()) {
-            return true;
-        }
-        if (content.is_array() && !content.empty()) {
-            return true;
-        }
-        return false;
-    }
-
-    static bool has_tool_calls(const json & msg) {
-        return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();
-    }
-};
-
-void convert_tool_responses_gemma4(json & messages) {
-    json result = json::array();
-    size_t i = 0;
-
-    while (i < messages.size()) {
-        auto & msg = messages[i];
-
-        if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||
-            !msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {
-            result.push_back(msg);
-            i++;
-            continue;
-        }
-
-        gemma4_model_turn_builder builder(messages, i);
-        result.push_back(builder.build());
-        i = builder.pos;
-    }
-
-    messages = result;
-}
-
-}
-
-common_chat_params common_chat_params_init_gemma4(const common_chat_template &    tmpl,
-                                                         const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-
-    if (inputs.add_generation_prompt && string_ends_with(data.prompt, "<turn|>\n")) {
-        // This may happen if the model generates content + tool_call, the
-        // template does not add the model's next turn and confuses the model
-        // from emitting its proper reasoning token sequence.
-        data.generation_prompt = "<|turn>model\n";
-        data.prompt += data.generation_prompt;
-    }
-
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_USER,      "<|turn>user"  },
-        { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" },
-    };
-
-    data.format            = COMMON_CHAT_FORMAT_PEG_GEMMA4;
-    data.supports_thinking  = true;
-    data.thinking_start_tag = "<|channel>thought";
-    data.thinking_end_tags  = {"<channel|>"};
-
-    data.preserved_tokens = {
-        "<|channel>",
-        "<channel|>",
-        "<|tool_call>",
-        "<tool_call|>",
-        "<|turn>",
-    };
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = string_ends_with(data.prompt, "<turn|>\n") ? "<|turn>model\n" : "";
-        data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += "<channel|>" + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
-
-        if (extract_reasoning) {
-            p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));
-        } else {
-            p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));
-        }
-
-        auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");
-        auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));
-
-        if (has_response_format) {
-            auto response_format = p.literal("```json") <<
-                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) <<
-                p.literal("```");
-            return start + p.optional(thought) + response_format;
-        }
-
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            // Gemma4 tool calling syntax
-            // Rules should match traversal logic in gemma4_to_json()
-            p.rule("gemma4-string-content", p.until("<|\"|>"));
-            p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));
-            p.rule("gemma4-bool", p.json_bool());
-            p.rule("gemma4-null", p.json_null());
-            p.rule("gemma4-number", p.json_number());
-            p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));
-            p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));
-            p.rule("gemma4-dict", [&]() {
-                auto ws = p.space();
-                auto member = p.ref("gemma4-dict-kv");
-                auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
-                return p.sequence({
-                    p.literal("{"), ws,
-                    p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
-                });
-            });
-            p.rule("gemma4-array", [&]() {
-                auto ws = p.space();
-                auto value = p.ref("gemma4-value");
-                auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
-                return p.sequence({
-                    p.literal("["), ws,
-                    p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
-                });
-            });
-            p.rule("gemma4-value", [&]() {
-                return p.choice({
-                    p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
-                    p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
-                });
-            });
-
-            auto tool_choice = p.choice();
-
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                std::string  name     = function.at("name");
-                // TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules
-                // const auto & params   = function.at("parameters");
-
-                tool_choice |= p.rule("tool-" + name, p.tool(p.sequence({
-                    p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),
-                    p.tool_args(p.ref("gemma4-dict")),
-                })));
-            });
-
-            auto tool_call = p.trigger_rule("tool-call", p.repeat(
-                "<|tool_call>call:" + tool_choice + "<tool_call|>",
-                /* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,
-                /* max = */ inputs.parallel_tool_calls ? -1 : 1
-            ));
-
-            auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));
-            auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));
-            auto message = p.rule("message", thought + content);
-            return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;
-        }
-
-        // Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may
-        // also emit a single trailing <channel|> token. Consume all complete reasoning blocks and
-        // then stop at the first unmatched <channel|> token.
-        auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"})));
-        auto message = p.rule("message", thought + content);
-        return start + p.one_or_more(message);
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" },
-        };
-    }
-
-    return data;
-}

common/parsers/gigachat-v3.cpp

@@ -1,76 +0,0 @@
-#include "parsers.h"
-
-common_chat_params common_chat_params_init_gigachat_v3(
-        const common_chat_template & tmpl,
-        const autoparser::generation_params & inputs) {
-
-    common_chat_params data;
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking = false;
-    data.preserved_tokens  = {
-        "<|message_sep|>\n\n",
-        "<|role_sep|>\n",
-    };
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-        data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content();
-        data.prompt += data.generation_prompt;
-    }
-
-    auto has_tools         = inputs.tools.is_array() && !inputs.tools.empty();
-    auto include_grammar   = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-    const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto ret = p.eps();
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            // Build a choice of all available tools
-            auto tool_choice = p.choice();
-            for (const auto & tool : inputs.tools) {
-                const auto & function = tool.at("function");
-                std::string name = function.at("name");
-                const auto  schema = common_chat_tool_parameters(function);
-
-                auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\"");
-                auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));
-
-                auto tool_open = p.tool_open(p.literal("{") << tool_name);
-
-                tool_choice |= p.rule("tool-" + name, tool_open << "," << tool_args << "}");
-            }
-
-            // Define the tool call structure
-            auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
-            auto max_calls = 1; // parallel toolcalls are not supported
-            auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice);
-            auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls));
-
-            ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls;
-        } else {
-            // Content only parser
-            include_grammar = false;
-            ret = p.content(p.rest());
-        }
-
-        return p.literal("assistant<|role_sep|>\n") + ret;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-
-        data.grammar = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            {COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix}
-        };
-    }
-    return data;
-}

common/parsers/gpt-oss.cpp

@@ -1,158 +0,0 @@
-#include "parsers.h"
-
-common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &    tmpl,
-                                                          const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    // Copy reasoning to the "thinking" field as expected by the gpt-oss template
-    auto adjusted_messages = json::array();
-    for (auto msg : inputs.messages) {
-        if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
-            msg["thinking"] = msg.at("reasoning_content");
-            if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
-                msg.erase("content");
-            }
-        }
-        adjusted_messages.push_back(msg);
-    }
-
-    auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
-
-    // Check if we need to replace the return token with end token during
-    // inference and without generation prompt. For more details see:
-    // https://github.com/ggml-org/llama.cpp/issues/15417
-    if (inputs.is_inference && !inputs.add_generation_prompt) {
-        static constexpr std::string_view return_token = "<|return|>";
-        static constexpr std::string_view end_token    = "<|end|>";
-        if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) {
-            prompt.replace(pos, return_token.length(), end_token);
-        }
-    }
-
-    data.prompt            = prompt;
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
-        { COMMON_CHAT_ROLE_USER,      "<|start|>user"      },
-        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>developer" },
-        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>system"    },
-        { COMMON_CHAT_ROLE_TOOL,      "<|start|>functions" },
-    };
-
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking = true;
-
-    data.thinking_start_tag = "<|channel|>analysis<|message|>";
-    data.thinking_end_tags  = {"<|end|>"};
-
-    // These special tokens are required to parse properly, so we include them
-    // even if parse_tool_calls is false.
-    data.preserved_tokens = {
-        "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
-    };
-
-    // Adjust prompt for continuation
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto start           = p.rule("start", p.literal("<|start|>assistant"));
-        auto end             = p.rule("end", p.literal("<|end|>"));
-        auto content         = p.rule("message-content", p.until("<|end|>"));
-        auto channel         = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
-        auto constrain_type  = p.chars("[A-Za-z0-9_-]", 1, -1);
-
-        // Occasionally, gpt-oss-20b will prefix channels with this commentary
-        auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));
-        auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>");
-
-        if (extract_reasoning) {
-            p.rule("analysis", start_analysis + p.reasoning(content) + end);
-        } else {
-            p.rule("analysis", p.content(start_analysis + content + end));
-        }
-
-        auto analysis = p.ref("analysis");
-        auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);
-        auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));
-
-        // Consume any unsolicited tool calls, e.g. builtin functions
-        auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));
-
-        auto any = p.rule("any", preamble | analysis);
-
-        if (has_response_format) {
-            auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
-            auto response_format = p.rule("response-format",
-                p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
-                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
-
-            return p.zero_or_more(start + analysis) + start + response_format;
-        }
-
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            auto tool_choice = p.choice();
-
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                std::string  name     = function.at("name");
-                const auto   params   = common_chat_tool_parameters(function);
-
-                auto func_name  = p.literal(" to=functions.") + p.tool_name(p.literal(name));
-                auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
-                auto args       = p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", params));
-
-                // recipient in role header
-                //   <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS
-                auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args);
-
-                // recipient in channel header
-                //   <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS
-                auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args);
-
-                tool_choice |= p.rule("tool-" + name, tool_in_role | tool_in_channel);
-            });
-
-            auto tool_call  = p.trigger_rule("tool-call", tool_choice);
-
-            if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
-                return p.zero_or_more(start + any) + start + tool_call;
-            }
-
-            return p.zero_or_more(start + any) + start + (tool_call | final_msg);
-        }
-
-        return p.zero_or_more(start + any) + start + (final_msg | unsolicited);
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
-            { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" },
-            { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
-            { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" }
-        };
-    }
-
-    return data;
-}

common/parsers/kimi-k2.cpp

@@ -1,128 +0,0 @@
-#include "parsers.h"
-
-// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>
-// The ID contains both the function name and an incrementing counter
-common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &    tmpl,
-                                                          const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking = true;
-    data.preserved_tokens  = {
-        "<|tool_calls_section_begin|>",
-        "<|tool_calls_section_end|>",
-        "<|tool_call_begin|>",
-        "<|tool_call_argument_begin|>",
-        "<|tool_call_end|>",
-        "<think>",
-        "</think>",
-    };
-
-    auto has_tools         = inputs.tools.is_array() && !inputs.tools.empty();
-    auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar   = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-
-    const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>";
-    const std::string SECTION_END   = "<|tool_calls_section_end|>";
-    const std::string CALL_BEGIN    = "<|tool_call_begin|>";
-    const std::string ARGS_BEGIN    = "<|tool_call_argument_begin|>";
-    const std::string CALL_END      = "<|tool_call_end|>";
-
-    const std::string THINK_START = "<think>";
-    const std::string THINK_END   = "</think>";
-    const std::string GEN_PROMPT  = "<|im_assistant|>assistant<|im_middle|>";
-
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = {THINK_END};
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += THINK_END + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        // Kimi K2 Thinking format:
-        // - Reasoning: <think>{reasoning}</think>
-        // - Content: text after reasoning
-        // - Tool calls section:
-        //   <|tool_calls_section_begin|>
-        //   <|tool_call_begin|>functions.<name>:<index><|tool_call_argument_begin|>{json_args}<|tool_call_end|>
-        //   ...
-        //   <|tool_calls_section_end|>
-        // The ID format is: functions.<function_name>:<counter> where counter is 0, 1, 2, ...
-
-        // Tool call markers
-        auto end = p.end();
-
-        // Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny.
-        // For example, it can call tools at the end of reasoning without closing reasoning...
-        auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning(
-            p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) +
-            p.optional(p.literal(THINK_END))) : p.eps();
-        auto generation_prompt = p.literal(GEN_PROMPT);
-
-
-        // Content only parser (no tools)
-        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-            return generation_prompt + reasoning + p.content(p.rest()) + end;
-        }
-
-        // Build tool call parsers for each available function
-        // The ID format is: functions.<name>:<index>
-        // We need to match: functions.<name>:<digits>
-        auto tool_choice = p.choice();
-        foreach_function(inputs.tools, [&](const json & tool) {
-            const auto & function = tool.at("function");
-            std::string  name     = function.at("name");
-            const auto   schema   = common_chat_tool_parameters(function);
-
-            // Match: functions.<name>:<digits>
-            // Capture the full call id (functions.<name>:<digits>) using tool_id tag
-            auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1));
-            auto tool_parser = p.tool(
-                p.tool_open(tool_id + p.literal(ARGS_BEGIN)) +
-                p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)) +
-                p.tool_close(p.optional((p.literal(CALL_END))))
-            );
-
-            tool_choice |= p.rule("tool-" + name, tool_parser);
-        });
-
-        // Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|>
-        auto min_calls  = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
-        auto max_calls  = inputs.parallel_tool_calls ? -1 : 1;
-        // Use trigger_rule so grammar generator knows where to start generating rules
-        auto tool_calls = p.rule("tool-calls",
-            p.optional(p.literal(SECTION_BEGIN)) +
-            p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) +
-                p.optional(p.literal(SECTION_END)))
-        );
-
-        auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN }));
-
-        return generation_prompt + reasoning + content_before_tools + tool_calls + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" }
-        };
-    }
-
-    return data;
-}

common/parsers/kimi-k3.cpp

@@ -1,167 +0,0 @@
-#include "parsers.h"
-
-// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros:
-//   open_tag(t, attrs) = <|open|>t k="v"...<|sep|>   close_tag(t) = <|close|>t<|sep|>
-//   assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|>
-// the generation prompt already opens the think (or response) section, so the
-// section opener is optional here - same as Kimi K2 Thinking
-common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &          tmpl,
-                                                          const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking = true;
-
-    const std::string SEP         = "<|sep|>";
-    const std::string MSG_START   = "<|open|>message role=\"assistant\"<|sep|>";
-    const std::string THINK_START = "<|open|>think<|sep|>";
-    const std::string THINK_END   = "<|close|>think<|sep|>";
-    const std::string RESP_START  = "<|open|>response<|sep|>";
-    const std::string RESP_END    = "<|close|>response<|sep|>";
-    const std::string TOOLS_START = "<|open|>tools<|sep|>";
-    const std::string TOOLS_END   = "<|close|>tools<|sep|>";
-    const std::string CALL_START  = "<|open|>call tool=\"";
-    const std::string CALL_END    = "<|close|>call<|sep|>";
-    const std::string ARG_START   = "<|open|>argument key=\"";
-    const std::string ARG_END     = "<|close|>argument<|sep|>";
-    const std::string MSG_END     = "<|close|>message<|sep|>";
-    const std::string EOM_TOKEN   = "<|end_of_msg|>";
-
-    // only the markers are special tokens. tag names ("think", "response", ...) are
-    // normal tokens and must not be preserved, or prose with those words is broken
-    data.preserved_tokens = {
-        "<|open|>",
-        "<|close|>",
-        "<|sep|>",
-        "<|end_of_msg|>",
-    };
-
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = { THINK_END };
-
-    // per-role message-start delimiters. user/assistant messages only have the role
-    // attribute, so the full opener is used. system and tool messages have more
-    // attributes, so those delimiters stop after the closing quote of the role
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" },
-        { COMMON_CHAT_ROLE_USER,      "<|open|>message role=\"user\"<|sep|>"      },
-        { COMMON_CHAT_ROLE_TOOL,      "<|open|>message role=\"tool\""             },
-        { COMMON_CHAT_ROLE_SYSTEM,    "<|open|>message role=\"system\""           },
-    };
-
-    auto has_tools         = inputs.tools.is_array() && !inputs.tools.empty();
-    auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar   = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += THINK_END + RESP_START + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto end = p.end();
-
-        auto start = p.optional(p.literal(MSG_START));
-
-        // the think section is always consumed, even with reasoning extraction off:
-        // the generation prompt ends with open_tag('think'), so it is always present.
-        // reasoning stops at its own closer, or at the response opener if the model
-        // skips the closer
-        auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) :
-                                              p.content(p.until_one_of({ THINK_END, RESP_START }));
-
-        auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body +
-                                    p.optional(p.literal(THINK_END)));
-
-        // content runs to the response closer, or to the next section if truncated
-        auto response = p.optional(p.literal(RESP_START)) +
-                        p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) +
-                        p.optional(p.literal(RESP_END));
-
-        // the EOG token after the message closer reaches the parser as text,
-        // so it must be consumed or the parse stays incomplete
-        auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN));
-
-        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-            return start + reasoning + response + trailer + end;
-        }
-
-        auto tool_choices = p.choice();
-        foreach_function(inputs.tools, [&](const json & tool) {
-            const auto & function = tool.at("function");
-            std::string  name     = function.at("name");
-            const json   schema   = common_chat_tool_parameters(function);
-
-            // arguments come one tag per key, with the JSON type in a type="..."
-            // attribute. the type is taken from the tool schema instead, as it tells
-            // us if the value is JSON or a literal string
-            auto args = p.eps();
-            if (schema.contains("properties") && !schema.at("properties").empty()) {
-                auto arg_choices = p.choice();
-                for (const auto & prop : schema.at("properties").items()) {
-                    const std::string & key = prop.key();
-
-                    std::string type = "string";
-                    if (prop.value().is_object() && prop.value().contains("type") &&
-                        prop.value().at("type").is_string()) {
-                        type = prop.value().at("type").get<std::string>();
-                    }
-
-                    auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) :
-                                                    p.tool_arg_value(p.until(ARG_END));
-
-                    // skip the trailing type="..." attribute: anything up to <|sep|>
-                    arg_choices |= p.rule("kimi-k3-arg-" + name + "-" + key,
-                                          p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) +
-                                                     p.tool_arg_name(p.literal(key)) + p.literal("\"") +
-                                                     p.until(SEP) + p.literal(SEP) + value +
-                                                     p.tool_arg_close(p.literal(ARG_END))));
-                }
-                args = p.zero_or_more(arg_choices);
-            }
-
-            // skip the trailing index="N" attribute the same way
-            auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") +
-                                           p.until(SEP) + p.literal(SEP)) +
-                               p.tool_args(args) + p.tool_close(p.literal(CALL_END)));
-
-            tool_choices |= p.rule("kimi-k3-tool-" + name, call);
-        });
-
-        // all calls go inside one tools section, then the message is closed. the
-        // message closer is part of the trigger rule, or else the lazy grammar
-        // rejects it once tool calls have started
-        auto tools_section =
-            p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) +
-                                                    p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) +
-                                                    p.optional(p.literal(EOM_TOKEN)));
-
-        auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section :
-                                                                              p.optional(tools_section);
-
-        return start + reasoning + response + tools + trailer + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START },
-        };
-    }
-
-    return data;
-}

common/parsers/lfm2.cpp

@@ -1,110 +0,0 @@
-#include "parsers.h"
-
-// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list
-// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call
-bool is_lfm2_template(const std::string & src) {
-    return src.find("<|tool_list_start|>") != std::string::npos &&
-           src.find("<|tool_list_end|>")   != std::string::npos;
-}
-
-// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable
-// (except dotted names and JSON literals true/false/null).
-// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning.
-// tool_list_tokens preserves LFM2 system tool-list markers.
-common_chat_params common_chat_params_init_lfm2(const common_chat_template &          tmpl,
-                                                       const autoparser::generation_params & inputs,
-                                                       bool tool_list_tokens) {
-    common_chat_params data;
-
-    const std::string TOOL_CALL_START = "<|tool_call_start|>";
-    const std::string TOOL_CALL_END   = "<|tool_call_end|>";
-    const std::string TOOL_LIST_START = "<|tool_list_start|>";
-    const std::string TOOL_LIST_END   = "<|tool_list_end|>";
-    const std::string THINK_START     = "<think>";
-    const std::string THINK_END       = "</think>";
-    const std::string GEN_PROMPT      = "<|im_start|>assistant\n";
-
-    // Copy reasoning to the "thinking" field the template expects
-    auto adjusted_messages = json::array();
-    for (auto msg : inputs.messages) {
-        if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
-            msg["thinking"] = msg.at("reasoning_content");
-        }
-        adjusted_messages.push_back(msg);
-    }
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking = true;
-    data.preserved_tokens  = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END };
-    if (tool_list_tokens) {
-        data.preserved_tokens.push_back(TOOL_LIST_START);
-        data.preserved_tokens.push_back(TOOL_LIST_END);
-    }
-
-    data.thinking_start_tag = THINK_START;
-    data.thinking_end_tags  = {THINK_END};
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
-    // Gate by reasoning format and whether the template supports <think>
-    auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE &&
-                             tmpl.source().find(THINK_START) != std::string::npos;
-    auto include_grammar   = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += THINK_END + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal(GEN_PROMPT);
-        auto end = p.end();
-
-        auto reasoning = p.eps();
-        if (extract_reasoning) {
-            reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
-        }
-
-        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-            if (has_response_format) {
-                auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));
-                return generation_prompt + reasoning + response_format + end;
-            }
-            return generation_prompt + reasoning + p.content(p.rest()) + end;
-        }
-        auto tool_calls = p.rule("tool-calls",
-            p.trigger_rule("tool-call",
-                p.literal(TOOL_CALL_START) +
-                p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) +
-                p.literal(TOOL_CALL_END)
-            )
-        );
-
-        auto content = p.content(p.until(TOOL_CALL_START));
-
-        return generation_prompt + reasoning + content + tool_calls + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START }
-        };
-    }
-
-    return data;
-}

common/parsers/minicpm5.cpp

@@ -1,130 +0,0 @@
-#include "parsers.h"
-
-// MiniCPM5 format:
-// - Reasoning: <think>{reasoning}</think> (optional)
-// - Tool calls: <function name="foo"><param name="bar">value</param></function>
-common_chat_params common_chat_params_init_minicpm5(const common_chat_template &          tmpl,
-                                                           const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking = true;
-    data.preserved_tokens  = {
-        "<function",
-        "<param",
-        "</function>",
-        "</param>",
-        "<think>",
-        "</think>",
-    };
-
-    data.thinking_start_tag = "<think>";
-    data.thinking_end_tags  = {"</think>"};
-
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant"             },
-        { COMMON_CHAT_ROLE_TOOL,      "<|im_start|>user\n<tool_response>" },
-        { COMMON_CHAT_ROLE_USER,      "<|im_start|>user"                  },
-        { COMMON_CHAT_ROLE_SYSTEM,    "<|im_start|>system"                },
-    };
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += "\n</think>\n\n" + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal("<|im_start|>assistant\n");
-
-        auto reasoning = p.eps();
-        if (extract_reasoning) {
-            reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space();
-        }
-
-        // Response format parser
-        if (has_response_format) {
-            return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema));
-        }
-
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            // CDATA lets a value carry characters that would otherwise close the tag (e.g.
-            // </param>); capture the inner text only, excluding the CDATA markers.
-            auto string_value = p.choice({
-                p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")),
-                p.negate(p.literal("<![CDATA[")) + p.ac(p.tool_arg_string_value(p.until("</param>")) + p.tool_arg_close(p.literal("</param>")), "</param>")
-            });
-
-            auto tool_choice = p.choice();
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto &      function = tool.at("function");
-                const std::string name     = function.at("name");
-
-                std::vector<common_peg_parser> arg_rules;
-                foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
-                    auto value_parser = p.eps();
-                    if (prop.schema->may_be_string()) {
-                        value_parser = string_value;
-                    } else {
-                        value_parser = p.tool_arg_json_value(
-                                p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema)
-                            ) + p.tool_arg_close(p.literal("</param>"));
-                    }
-
-                    arg_rules.push_back(p.tool_arg(
-                        p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
-                        value_parser
-                    ));
-                });
-
-                auto args = p.eps();
-                if (!arg_rules.empty()) {
-                    args = p.zero_or_more(p.choice(arg_rules) + p.space());
-                }
-
-                auto tool_parser = p.tool(
-                    p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">"))
-                    << p.tool_args(args)
-                    << p.tool_close(p.literal("</function>")));
-
-                tool_choice |= p.rule("tool-" + name, tool_parser);
-            });
-
-            auto max_calls  = inputs.parallel_tool_calls ? -1 : 1;
-            auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls));
-
-            auto content = p.content(p.until("<function"));
-
-            return generation_prompt + reasoning + content + tool_calls + p.end();
-        }
-
-        return generation_prompt + reasoning + p.content(p.rest()) + p.end();
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },
-        };
-    }
-
-    return data;
-}

common/parsers/minimax-m3.cpp

@@ -1,229 +0,0 @@
-#include "parsers.h"
-
-common_chat_params common_chat_params_init_minimax_m3(const common_chat_template &          tmpl,
-                                                             const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    data.prompt             = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt  = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format             = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3;
-    data.supports_thinking  = true;
-    data.thinking_start_tag = "<mm:think>";
-    data.thinking_end_tags  = {"</mm:think>"};
-
-    // M3 prefixes every tool tag with the namespace token "]<]minimax[>[";
-    // params use the parameter name as the tag (<file_path>...</file_path>).
-    const std::string NS          = "]<]minimax[>[";
-    const std::string THINK_START = "<mm:think>";
-    const std::string THINK_END   = "</mm:think>";
-    const std::string FC_START    = NS + "<tool_call>";
-    const std::string FC_END      = NS + "</tool_call>";
-    const std::string INVOKE_END  = NS + "</invoke>";
-
-    data.preserved_tokens = {
-        NS,
-        "<tool_call>",
-        "</tool_call>",
-        THINK_START,
-        THINK_END,
-    };
-
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai"        },
-        { COMMON_CHAT_ROLE_USER,      "]~b]user"      },
-        { COMMON_CHAT_ROLE_TOOL,      "]~b]tool"      },
-        { COMMON_CHAT_ROLE_SYSTEM,    "]~b]developer" },
-        { COMMON_CHAT_ROLE_SYSTEM,    "]~b]system"    },
-    };
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    const std::string GEN_PROMPT = data.generation_prompt;
-
-    using mm3 = common_chat_peg_minimax_m3_mapper;
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += THINK_END + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);
-        auto end = p.end();
-
-        auto reasoning = p.eps();
-        if (extract_reasoning) {
-            auto block = inputs.enable_thinking
-                             ? p.literal(THINK_START) + p.space() +
-                                   p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END)
-                             : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END);
-
-            // A turn without reasoning is prefixed with a bare </mm:think>, written either by the
-            // generation prompt (thinking_mode = "disabled") or by the model itself.
-            reasoning = p.optional(p.choice({ block, p.literal(THINK_END) }));
-        }
-
-        if (has_response_format) {
-            auto response_format = p.rule("response-format",
-                p.literal("```json") + p.space() +
-                p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
-                p.space() + p.literal("```"));
-            return generation_prompt + reasoning + response_format + end;
-        }
-
-        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
-            return generation_prompt + reasoning + p.content(p.rest()) + end;
-        }
-
-        auto tool_choice = p.choice();
-        foreach_function(inputs.tools, [&](const json & tool) {
-            const auto & function = tool.at("function");
-            std::string  name     = function.at("name");
-            auto         params   = common_chat_tool_parameters(function);
-            auto         doc      = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
-
-            // The template expands argument values recursively in XML (see the to_xml() macro)
-            std::function<common_peg_parser(const common_chat_schema &, const std::string &, const std::string &)> value_of;
-            std::function<common_peg_parser(const common_chat_schema_object &, const std::string &)>              members_of;
-
-            auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) {
-                const std::string close = NS + "</" + tag + ">";
-                return p.rule(rule_name,
-                    p.tool_arg(
-                        p.tool_arg_open(
-                            p.literal(NS + "<") +
-                            p.tool_arg_name(p.literal(tag)) +
-                            p.literal(">")) +
-                        value_of(schema, rule_name, close)));
-            };
-
-            value_of = [&](const common_chat_schema & schema,
-                           const std::string & rule_name,
-                           const std::string & close) -> common_peg_parser {
-                auto close_tag = p.tool_arg_close(p.literal(close));
-
-                // A string accepts anything, so a union with a string alternative is a string
-                if (schema.may_be_string()) {
-                    return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);
-                }
-
-                if (schema.kind() == common_chat_schema::KIND_ANY_OF) {
-                    std::vector<common_peg_parser> choices;
-
-                    size_t index = 0;
-                    for (const auto & alternative : static_cast<const common_chat_schema_any_of &>(schema).children) {
-                        const std::string alt_name = rule_name + "-" + std::to_string(index++);
-
-                        // There is a risk that this breaks streaming deltas, but that's a risk we
-                        // assume to provide tool arg streaming.
-                        choices.push_back(value_of(*alternative, alt_name, close));
-                    }
-
-                    return p.choice(choices);
-                }
-
-                if (schema.kind() == common_chat_schema::KIND_OBJECT) {
-                    const auto & object = static_cast<const common_chat_schema_object &>(schema);
-                    if (!object.properties.empty()) {
-                        return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag;
-                    }
-                }
-
-                if (schema.kind() == common_chat_schema::KIND_ARRAY) {
-                    const std::string item_close = NS + "</item>";
-                    auto item = p.rule(rule_name + "-item",
-                        p.tag(mm3::TOOL_ARG_ITEM,
-                              p.literal(NS + "<item>") +
-                                  value_of(*static_cast<const common_chat_schema_array &>(schema).items, rule_name + "-item", item_close)));
-                    return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;
-                }
-
-                return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, schema)) + close_tag;
-            };
-
-            // Required properties in schema order, then any number of optional ones in any order.
-            members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser {
-                std::vector<common_peg_parser> required_elements;
-                std::vector<common_peg_parser> optional_elements;
-                for (const auto & prop : object.properties) {
-                    auto element = element_of(prop.name, *prop.schema, rule_prefix + "-" + prop.name);
-                    (prop.required ? required_elements : optional_elements).push_back(element);
-                }
-
-                common_peg_parser members = p.eps();
-                for (size_t i = 0; i < required_elements.size(); i++) {
-                    if (i > 0) {
-                        members = members + p.space();
-                    }
-                    members = members + required_elements[i];
-                }
-
-                if (!optional_elements.empty()) {
-                    common_peg_parser any_optional = p.choice();
-                    for (const auto & element : optional_elements) {
-                        any_optional |= element;
-                    }
-                    members = members + p.repeat(p.space() + any_optional, 0, -1);
-                }
-
-                return members;
-            };
-
-            common_peg_parser invoke_body = p.eps();
-            if (doc->root->kind() == common_chat_schema::KIND_OBJECT) {
-                invoke_body = members_of(static_cast<const common_chat_schema_object &>(*doc->root), "tool-" + name + "-arg");
-            }
-
-            auto func_parser = p.tool(
-                p.tool_open(p.literal(NS + "<invoke name=\"") +
-                            p.tool_name(p.literal(name)) + p.literal("\">")) +
-                p.space() + invoke_body + p.space() +
-                p.tool_close(p.literal(INVOKE_END)));
-
-            tool_choice |= p.rule("tool-" + name, func_parser);
-        });
-
-        auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-
-        common_peg_parser tool_calls = p.eps();
-        if (inputs.parallel_tool_calls) {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice +
-                p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
-        } else {
-            tool_calls = p.trigger_rule("tool-call",
-                p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
-        }
-
-        if (!require_tools) {
-            tool_calls = p.optional(tool_calls);
-        }
-
-        auto content_before_tools = p.content(p.until(FC_START));
-        return generation_prompt + reasoning + content_before_tools + tool_calls + end;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
-        };
-    }
-
-    return data;
-}

common/parsers/ministral3.cpp

@@ -1,126 +0,0 @@
-#include "parsers.h"
-
-common_chat_params common_chat_params_init_ministral_3(const common_chat_template &    tmpl,
-                                                              const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    // Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja
-    auto adjusted_messages = json::array();
-    for (const auto & msg : inputs.messages) {
-        auto role = msg.value("role", "");
-        if (role != "system" && role != "assistant") {
-            // Only adjust system and assistant messages. Interestingly, the system message may contain thinking.
-            adjusted_messages.push_back(msg);
-            continue;
-        }
-
-        auto content = json::array();
-
-        // If message contains `reasoning_content`, add it as a block of type `thinking`
-        if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
-            content.push_back({
-                { "type",     "thinking"                                     },
-                { "thinking", msg.at("reasoning_content").get<std::string>() },
-            });
-        }
-
-        // If message contains `content`, add it as a block of type `text`
-        if (msg.contains("content")) {
-            if (msg.at("content").is_string()) {
-                content.push_back({
-                    { "type", "text"                               },
-                    { "text", msg.at("content").get<std::string>() },
-                });
-            } else if (msg.at("content").is_array()) {
-                auto blocks = msg.at("content");
-                content.insert(blocks);
-            }
-        }
-
-        auto adjusted       = msg;
-        adjusted["content"] = content;
-        adjusted.erase("reasoning_content");
-        adjusted_messages.push_back(adjusted);
-    }
-
-    auto has_tools            = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format  = inputs.json_schema.is_object() && !inputs.json_schema.empty();
-    auto extract_reasoning    = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar      = true;
-
-    data.supports_thinking  = true;
-    data.thinking_start_tag = "[THINK]";
-    data.thinking_end_tags  = {"[/THINK]"};
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.preserved_tokens  = {
-        "[THINK]",
-        "[/THINK]",
-        "[TOOL_CALLS]",
-        "[ARGS]",
-    };
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = "[THINK]" + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += "[/THINK]" + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.eps();
-        auto reasoning =
-            extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
-
-        // Response format parser
-        if (has_response_format) {
-            // Ministral wants to emit json surrounded by code fences
-            return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```");
-        }
-
-        // Tool call parser
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            auto tool_choice = p.choice();
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                std::string  name     = function.at("name");
-                const auto   schema   = common_chat_tool_parameters(function);
-
-                tool_choice |=
-                    p.rule("tool-" + name, p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") +
-                                               p.tool_args(p.schema(p.json(), "tool-" + name + "-schema", schema)));
-            });
-
-            auto min_calls  = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
-            auto max_calls  = inputs.parallel_tool_calls ? -1 : 1;
-            auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));
-
-            return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls);
-        }
-
-        // Content only parser
-        include_grammar = false;
-        return generation_prompt + (reasoning << p.content(p.rest()));
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-
-        data.grammar = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" }
-        };
-    }
-
-    return data;
-}

common/parsers/muse-glimmer.cpp

@@ -1,138 +0,0 @@
-#include "parsers.h"
-
-// An assistant turn is rendered as one or more messages, each
-// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is
-// <|eom|> (more messages follow) or <|eot|> (end of turn):
-//   - chain-of-thought: to=self, terminated by <|eom|>
-//   - final answer:     to=user, terminated by <|eot|>
-// The generation prompt is just "<|start|>assistant"; the model emits its own
-// " to=...<|message|>".
-common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template &          tmpl,
-                                                               const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = "<|start|>assistant";
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-    data.supports_thinking = true;
-
-    data.preserved_tokens = {
-        "<|start|>", "<|message|>", "<|eom|>", "<|eot|>",
-        // ATEM tool-call markup emitted on " to=<tool>" turns.
-        "<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",
-        "</atem:invoke>", "</atem:function_calls>",
-    };
-
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
-        { COMMON_CHAT_ROLE_USER,      "<|start|>user"      },
-        { COMMON_CHAT_ROLE_SYSTEM,    "<|start|>system"    },
-        { COMMON_CHAT_ROLE_TOOL,      "<|start|>tool"      },
-    };
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-
-    auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
-    // Constrained grammar whenever tools are offered.
-    auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto start = p.rule("start", p.literal("<|start|>assistant"));
-
-        if (!extract_reasoning && !include_grammar) {
-            return start + p.content(p.rest());
-        }
-
-        if (extract_reasoning) {
-            p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
-        } else {
-            p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
-        }
-        auto analysis = p.ref("analysis");
-
-        auto recipient  = p.optional(p.literal(" to=user"));
-        auto final_msg  = p.rule("final", recipient + p.literal("<|message|>") +
-                                              p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
-
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            auto string_value = p.ac(
-                p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
-                "</atem:parameter>");
-
-            auto tool_choice = p.choice();
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto &      function = tool.at("function");
-                const std::string name     = function.at("name");
-
-                std::vector<common_peg_parser> arg_rules;
-                foreach_parameter(function, [&](const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
-                    auto value_parser = p.eps();
-                    if (prop.schema->may_be_string()) {
-                        value_parser = string_value;
-                    } else {
-                        value_parser = p.tool_arg_json_value(
-                                p.schema(p.json(), "tool-" + name + "-arg-" + prop.name + "-schema", doc, *prop.schema))
-                            + p.tool_arg_close(p.literal("</atem:parameter>"));
-                    }
-
-                    arg_rules.push_back(p.tool_arg(
-                        p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
-                        value_parser));
-                });
-
-                auto args = p.eps();
-                if (!arg_rules.empty()) {
-                    args = p.zero_or_more(p.choice(arg_rules) + p.space());
-                }
-
-                auto tool_parser = p.tool(
-                    p.tool_open(p.literal(" to=") + p.until("<|message|>") +
-                                p.literal("<|message|><atem:function_calls>") + p.space() +
-                                p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
-                    << p.tool_args(args)
-                    << p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
-
-                tool_choice |= p.rule("tool-" + name, tool_parser);
-            });
-
-            auto tool_calls = inputs.parallel_tool_calls
-                ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
-                : p.trigger_rule("tool-call", tool_choice);
-
-
-            if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
-                return p.zero_or_more(start + analysis) + start + tool_calls;
-            }
-            auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
-            return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));
-        }
-
-        return p.zero_or_more(start + analysis) + start + final_msg;
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
-        data.grammar      = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-        data.grammar_triggers = {
-            { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
-              "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
-        };
-    }
-
-    return data;
-}

common/parsers/parsers.cpp

@@ -1,25 +0,0 @@
-#include "parsers.h"
-
-#include "log.h"
-
-void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
-    for (const auto & tool : tools) {
-        if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
-            LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());
-            continue;
-        }
-        fn(tool);
-    }
-}
-
-void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn) {
-    auto         params = common_chat_tool_parameters(function);
-    auto         doc    = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
-    const auto * object = dynamic_cast<const common_chat_schema_object *>(doc->root.get());
-    if (!object) {
-        return;
-    }
-    for (const auto & prop : object->properties) {
-        fn(prop, doc);
-    }
-}

common/parsers/parsers.h

@@ -1,77 +0,0 @@
-#pragma once
-
-#include "chat.h"
-#include "chat-auto-parser.h"
-#include "chat-auto-parser-helpers.h"
-#include "chat-peg-parser.h"
-#include "common.h"
-#include "ggml.h"
-#include "json-schema-to-grammar.h"
-#include "json.h"
-
-#include <functional>
-#include <optional>
-#include <set>
-#include <string>
-#include <vector>
-
-using json = common_json;
-
-// iterate over the function tools of an OpenAI-style tools array
-void foreach_function(const json & tools, const std::function<void(const json &)> & fn);
-
-// iterate over the parameters of a function tool, with the document that owns them
-void foreach_parameter(const json & function, const std::function<void(const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn);
-
-// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten
-std::string common_chat_template_direct_apply_impl(
-    const common_chat_template & tmpl,
-    const autoparser::generation_params & inputs,
-    const std::optional<json> & messages_override = std::nullopt,
-    const std::optional<json> & tools_override = std::nullopt,
-    const std::optional<json> & additional_context = std::nullopt);
-
-// the suffix a template appends when add_generation_prompt is set
-std::string common_chat_template_generation_prompt_impl(
-    const common_chat_template & tmpl,
-    const autoparser::generation_params & inputs,
-    const std::optional<json> & messages_override = std::nullopt,
-    const std::optional<json> & tools_override = std::nullopt,
-    const std::optional<json> & additional_context = std::nullopt);
-
-bool is_lfm2_template(const std::string & src);
-
-namespace workaround {
-
-void convert_tool_responses_gemma4(json & messages);
-
-}
-
-common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_gigachat_v3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-// tool_list_tokens preserves the LFM2 system tool-list markers; LFM2.5 renders without them
-common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, const autoparser::generation_params & inputs, bool tool_list_tokens);
-
-common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
-
-common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs);

common/parsers/qwen3-coder.cpp

@@ -1,193 +0,0 @@
-#include "parsers.h"
-
-common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template &          tmpl,
-                                                              const autoparser::generation_params & inputs) {
-    common_chat_params data;
-
-    const std::string GEN_PREFIX = "<|im_start|>assistant\n";
-
-    data.prompt            = common_chat_template_direct_apply_impl(tmpl, inputs);
-    data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
-    data.format            = COMMON_CHAT_FORMAT_PEG_NATIVE;
-
-    auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;
-
-    data.supports_thinking = supports_reasoning;
-    data.preserved_tokens  = {
-        "<tool_call>",
-        "</tool_call>",
-    };
-
-    auto is_qwen3_coder  = !supports_reasoning;
-
-    if (supports_reasoning) {
-        data.thinking_start_tag = "<think>";
-        // Support both </think> and <tool_call> as reasoning end sequences.
-        // <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
-        data.thinking_end_tags = { "</think>", "<tool_call>" };
-        data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
-    }
-
-    data.message_delimiters = {
-        { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant"             },
-        { COMMON_CHAT_ROLE_TOOL,      "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3
-        { COMMON_CHAT_ROLE_TOOL,      "<|im_start|>tool_response"         }, // StepFun-3.5-Flash
-        { COMMON_CHAT_ROLE_USER,      "<|im_start|>user"                  },
-        { COMMON_CHAT_ROLE_SYSTEM,    "<|im_start|>system"                },
-    };
-
-    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
-    auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
-    auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-    auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
-
-    if (inputs.has_continuation()) {
-        const auto & msg = inputs.continue_msg;
-
-        data.generation_prompt = GEN_PREFIX;
-        if (supports_reasoning) {
-            data.generation_prompt += "<think>\n" + msg.reasoning_content;
-            if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-                data.generation_prompt += "\n</think>\n\n";
-            }
-        }
-        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
-            data.generation_prompt += msg.render_content();
-        }
-
-        data.prompt += data.generation_prompt;
-    }
-
-    std::vector<std::string> tool_call_starts = { "<tool_call>" };
-
-    if (is_qwen3_coder) {
-        // Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
-        // starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
-        // constraining on <function which may occur in valid content generation, e.g. #include <functional>
-        foreach_function(inputs.tools, [&](const json & tool) {
-            const std::string name = tool.at("function").at("name");
-            tool_call_starts.push_back("<function=" + name + ">");
-        });
-    }
-
-    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
-        auto generation_prompt = p.literal(GEN_PREFIX);
-
-        auto reasoning = p.eps();
-        if (supports_reasoning && extract_reasoning) {
-            reasoning = p.optional("<think>" + p.space() +
-                                   p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
-                                   (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
-        }
-
-        // Response format parser
-        if (has_response_format) {
-            return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
-        }
-
-        // Tool call parser
-        if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
-            auto arg_close  = p.tool_arg_close(p.literal("\n</parameter>\n"));
-            auto arg_string = p.rule("xml-arg-string",
-                p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
-
-            auto tool_choice = p.choice();
-            foreach_function(inputs.tools, [&](const json & tool) {
-                const auto & function = tool.at("function");
-                std::string  name     = function.at("name");
-
-                std::vector<common_peg_parser> required_args;
-                std::vector<common_peg_parser> optional_args;
-
-                foreach_parameter(function, [&](const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
-                    auto rule_name = "tool-" + name + "-arg-" + param.name;
-
-                    auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param.name)) + ">\n");
-
-                    auto types = param.schema->value_types();
-
-                    auto arg_value = p.eps();
-                    if (!types.has(common_chat_schema::TYPE_STRING)) {
-                        arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
-                    } else if (types.is_only(common_chat_schema::TYPE_STRING)) {
-                        arg_value = arg_string;
-                    } else {
-                        // The string alternative accepts any text, so the grammar only keeps the raw string
-                        // rule. The parser still tries the JSON alternatives first to type the value.
-                        auto json_value = p.choice();
-                        if (types.has(common_chat_schema::TYPE_OBJECT)) {
-                            json_value |= p.json_object();
-                        }
-                        if (types.has(common_chat_schema::TYPE_ARRAY)) {
-                            json_value |= p.json_array();
-                        }
-                        if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
-                            json_value |= p.json_number();
-                        }
-                        if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
-                            json_value |= p.json_bool();
-                        }
-                        if (types.has(common_chat_schema::TYPE_NULL)) {
-                            json_value |= p.json_null();
-                        }
-                        arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "xml-arg-string");
-                    }
-
-                    auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
-
-                    (param.required ? required_args : optional_args).push_back(arg_rule);
-                });
-
-                // Accept required arguments in any order, as Qwen does not always adhere to the
-                // order provided.
-                auto args = p.permute("tool-" + name + "-args", required_args);
-                if (!optional_args.empty()) {
-                    args = args + p.zero_or_more(p.choice(optional_args));
-                }
-
-                auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
-                                   p.tool_args(args) +
-                                   p.tool_close(p.literal("</function>\n")));
-
-                tool_choice |= p.rule("tool-" + name, func);
-            });
-
-            auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
-
-            auto tool_call_body = tool_choice + "</tool_call>" + p.space();
-            auto tool_call      = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
-
-            // Qwen3-Coder models may occasionally omit the <tool_call> token.
-            auto tool_call_first = is_qwen3_coder ?
-                p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :
-                tool_call;
-
-            auto calls      = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
-            auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
-
-            return generation_prompt +
-                   (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
-        }
-
-        // Content only parser
-        return generation_prompt + (reasoning << p.content(p.rest()));
-    });
-
-    data.parser = parser.save();
-
-    if (include_grammar) {
-        data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
-
-        data.grammar = build_grammar([&](const common_grammar_builder & builder) {
-            parser.build_grammar(builder, data.grammar_lazy);
-        });
-
-        if (data.grammar_lazy) {
-            for (const auto & start : tool_call_starts) {
-                data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });
-            }
-        }
-    }
-
-    return data;
-}

common/parsers/sources.cmake

@@ -1,20 +0,0 @@
-# Specialized chat template parsers, listed explicitly so that adding or removing one re-runs CMake instead of leaving an incremental build stale.
-
-set(LLAMA_CHAT_PARSERS_SOURCES
-    ${CMAKE_CURRENT_LIST_DIR}/parsers.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/parsers.h
-    ${CMAKE_CURRENT_LIST_DIR}/cohere2moe.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/deepseek.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/functionary-v3-2.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/lfm2.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/minicpm5.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp
-)

common/peg-parser.cpp

@@ -953,7 +953,7 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
         } else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
             return "Until(" + string_join(p.delimiters, " | ") + ")";
         } else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
-            return "Schema(" + dump_impl(p.child, visited) + ", " + (p.node ? common_chat_schema::kind_name(p.node->kind()) : "null") + ")";
+            return "Schema(" + dump_impl(p.child, visited) + ", " + (p.schema ? p.schema->dump() : "null") + ")";
         } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
             return "Rule(" + p.name + ", " + dump_impl(p.child, visited) + ")";
         } else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
@@ -1119,13 +1119,8 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes,
     return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
 }
 
-common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw) {
-    return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::move(doc), &node, raw}));
-}
-
 common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
-    auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(schema));
-    return this->schema(p, name, doc, *doc->root, raw);
+    return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<common_json>(schema), raw}));
 }
 
 common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
@@ -1578,9 +1573,30 @@ static std::set<std::string> collect_reachable_rules(
 
 // GBNF generation implementation
 void common_peg_arena::build_grammar(const common_grammar_builder & builder, bool lazy) const {
-    // A raw string value is parsed by the child rather than constrained by the schema
     auto schema_delegates = [](const common_peg_schema_parser & s) -> bool {
-        return !s.node || (s.raw && s.node->may_be_string());
+        if (!s.schema) {
+            return true;
+        }
+        if (s.raw && s.schema->contains("type")) {
+            const auto & type_val = s.schema->at("type");
+            if (type_val.is_string() && type_val == "string") {
+                return true;
+            }
+            // Handle nullable types like ["string", "null"] - delegate when the
+            // non-null type is string, since the tagged format uses raw text
+            if (type_val.is_array()) {
+                for (const auto & t : type_val) {
+                    if (t.is_string() && t.get<std::string>() != "null") {
+                        return t.get<std::string>() == "string";
+                    }
+                }
+            }
+        }
+        // Delegate for enum schemas in raw mode - enum values are literal strings
+        if (s.raw && !s.schema->contains("type") && s.schema->contains("enum")) {
+            return true;
+        }
+        return false;
     };
 
     // Unwrap the parser so we can properly check if it's a sequence or choice
@@ -1715,7 +1731,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
                 if (schema_delegates(p)) {
                     return to_gbnf(p.child);
                 }
-                return builder.add_schema(p.name, *p.node);
+                return builder.add_schema(p.name, *p.schema);
             } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
                 return p.name;
             } else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
@@ -1843,6 +1859,7 @@ static common_json serialize_parser_variant(const common_peg_parser_variant & va
                 {"type", "schema"},
                 {"child", p.child},
                 {"name", p.name},
+                {"schema", p.schema ? *p.schema : json(nullptr)},
                 {"raw", p.raw}
             };
         } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
@@ -1982,12 +1999,15 @@ static common_peg_parser_variant deserialize_parser_variant(const common_json &
         return common_peg_until_parser{j["delimiters"].get<std::vector<std::string>>()};
     }
     if (type == "schema") {
-        if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) {
+        if (!j.contains("child") || !j.contains("name") || !j.contains("schema") || !j.contains("raw")) {
             throw std::runtime_error("schema parser missing required fields");
         }
         common_peg_schema_parser parser;
         parser.child = j["child"].get<common_peg_parser_id>();
         parser.name = j["name"];
+        if (!j["schema"].is_null()) {
+            parser.schema = std::make_shared<common_json>(j["schema"]);
+        }
         parser.raw = j["raw"].get<bool>();
         return parser;
     }

common/peg-parser.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "json-schema.h"
 #include "json.h"
 
 #include <memory>
@@ -246,8 +245,7 @@ struct common_peg_until_parser {
 struct common_peg_schema_parser {
     common_peg_parser_id child;
     std::string name;
-    common_chat_schema_document_ptr doc;  // owns node
-    const common_chat_schema * node = nullptr;
+    std::shared_ptr<common_json> schema;
 
     // Indicates if the GBNF should accept a raw string that matches the schema.
     bool raw;
@@ -490,10 +488,8 @@ class common_peg_parser_builder {
     // A marker, i.e. text delimited by a pair of <> or []
     common_peg_parser marker();
 
-    // Wraps a parser with the schema its GBNF is generated from, a node of the document that owns it
-    common_peg_parser schema(const common_peg_parser & p, const std::string & name, common_chat_schema_document_ptr doc, const common_chat_schema & node, bool raw = false);
-
-    // Parses the JSON schema into a document of its own
+    // Wraps a parser with JSON schema metadata for grammar generation.
+    // Used internally to convert JSON schemas to GBNF grammar rules.
     common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false);
 
     // Creates a named rule, stores it in the grammar, and returns a ref.

common/speculative.cpp

@@ -14,7 +14,6 @@
 
 #include <algorithm>
 #include <cassert>
-#include <cmath>
 #include <cstring>
 #include <iomanip>
 #include <map>
@@ -139,7 +138,6 @@ struct common_speculative_impl {
     const common_speculative_type type;
 
     uint32_t n_seq;
-    int32_t n_max; // maximum draft length after implementation-specific limits
 
     size_t n_call_begin  = 0; // number of times this implementation was called for refresh.
     size_t n_call_draft  = 0; // number of times this implementation was called for generation.
@@ -159,7 +157,7 @@ struct common_speculative_impl {
     int64_t t_draft_us  = 0; // total time spent in generating drafts in this implementation in microseconds.
     int64_t t_accept_us = 0; // total time spent in accumulation of this implementation in microseconds.
 
-    common_speculative_impl(common_speculative_type type, uint32_t n_seq, int32_t n_max) : type(type), n_seq(n_seq), n_max(n_max) {}
+    common_speculative_impl(common_speculative_type type, uint32_t n_seq) : type(type), n_seq(n_seq) {}
 
     virtual ~common_speculative_impl() = default;
 
@@ -184,7 +182,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
     std::vector<common_sampler_ptr> smpls;
 
     common_speculative_impl_draft_simple(const common_params_speculative & params, uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq, params.draft.n_max)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq)
         , params(params.draft)
     {
         auto * ctx_dft = this->params.ctx_dft;
@@ -296,7 +294,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
             drafting[seq_id] = true;
             common_sampler_reset(smpls[seq_id].get());
 
-            common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true);
+            common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true);
         }
 
         int ret = llama_decode(ctx_dft, batch);
@@ -355,7 +353,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
                     continue;
                 }
 
-                common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true);
+                common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true);
             }
 
             if (batch.n_tokens == 0) {
@@ -454,7 +452,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
     std::vector<float> g_embd_buf;
 
     common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq, params.draft.n_max)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq)
         , params(params.draft)
     {
         SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n");
@@ -925,25 +923,21 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
     int32_t     block_size    = 0;
     llama_token mask_token_id = 0;
 
-    bool    is_dflash2     = false;
-    bool    is_mrope       = false;
-    int32_t selector_top_k = 0;
-
     // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
     const bool is_dspark;
 
     // dspark speculators
     bool sample_from_anchor = true;
 
-    // block-internal attention
-    bool causal_attn = false;
-
     const int32_t * target_layer_ids   = nullptr; // model_dft's extract layer indices
     uint32_t        target_layer_ids_n = 0;
 
+    // scratch buffer for concatenated target features [n_tokens, n_embd_enc]
+    std::vector<float> features_buf;
+
     common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
             common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
-        : common_speculative_impl(type, n_seq, params.draft.n_max)
+        : common_speculative_impl(type, n_seq)
         , params(params.draft)
         , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)
     {
@@ -972,25 +966,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             if (llama_model_meta_val_str(model_dft, "dflash.sample_from_anchor", buf, sizeof(buf)) >= 0) {
                 sample_from_anchor = std::strcmp(buf, "true") == 0;
             }
-            if (llama_model_meta_val_str(model_dft, "dflash.attention.causal", buf, sizeof(buf)) >= 0) {
-                causal_attn = std::strcmp(buf, "true") == 0;
-            }
         }
-
-        selector_top_k = llama_model_dflash_selector_top_k(model_dft);
-        is_dflash2     = selector_top_k > 0;
         mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft));
 
-        if (is_dspark && this->params.p_min > 0.0f) {
-            char buf[16] = {};
-            const bool has_conf =
-                llama_model_meta_val_str(model_dft, "dflash.has_confidence_head", buf, sizeof(buf)) < 0 ||
-                std::strcmp(buf, "true") == 0;
-            if (!has_conf) {
-                throw std::runtime_error("DSpark draft has no confidence head: please set --spec-draft-p-min 0");
-            }
-        }
-
         LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str());
         LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min);
         LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u, sample_from_anchor=%s\n", __func__,
@@ -1005,17 +983,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             this->params.n_max = std::min(this->params.n_max, n_draft_max);
             this->params.n_min = std::min(this->params.n_min, n_draft_max);
         }
-        this->n_max = this->params.n_max;
 
         batch        = llama_batch_init(llama_n_batch(ctx_dft), 0,          n_seq);
-        batch_inject = llama_batch_init(llama_n_ubatch(ctx_dft), n_embd_enc, n_seq);
-
-        // embd batches on an M-RoPE draft need 4 position rows per token
-        is_mrope = llama_model_rope_type(model_dft) == LLAMA_ROPE_TYPE_MROPE;
-        if (is_mrope) {
-            free(batch_inject.pos);
-            batch_inject.pos = (llama_pos *) malloc(sizeof(llama_pos) * 4 * llama_n_batch(ctx_dft));
-        }
+        batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq);
 
         smpls.resize(n_seq);
         for (auto & s : smpls) {
@@ -1028,7 +998,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
         // offload draft sampling to the backend
         backend_chains.assign(n_seq, nullptr);
-        if (this->params.backend_sampling && !is_dflash2) {
+        if (this->params.backend_sampling) {
             for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
                 llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
                 llama_sampler_chain_add(chain, llama_sampler_init_top_k(10));
@@ -1047,9 +1017,8 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
         }
 
-        // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
-        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2);
-        llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
+        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true);
+        llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention
     }
 
     ~common_speculative_impl_draft_dflash() override {
@@ -1094,7 +1063,8 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
         // Target prefill may contain token IDs or multimodal embeddings. Both
         // produce the target-layer features used to seed the draft KV cache, so
-        // embeddings are injected too, except the pinned ones skipped below.
+        // skipping the embedding batches leaves a hole in the draft's cache and
+        // the next injection fails to initialize.
         // TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged
         const bool has_tokens     = batch_in.token != nullptr;
         const bool has_embeddings = batch_in.embd  != nullptr;
@@ -1130,44 +1100,55 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             }
             const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1;
 
-            // an M-RoPE image pins all its rows to one position, so a windowed draft
-            // cache cannot free cells for it - skip it, the draft can jump over the gap
-            const bool pos_pinned = batch_in.pos[i_batch_beg[seq_id]] == batch_in.pos[i_batch_end[seq_id]];
-            if (has_embeddings && n_rows > 1 && pos_pinned) {
-                continue;
-            }
-
             for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
                 const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);
 
-                // gather target features per extract layer; the fused decode encodes and
-                // injects them into the K/V cache at the target positions
-                batch_inject.n_tokens = n_chunk;
+                // gather this chunk's target features, interleaved by extract layer
+                features_buf.resize((size_t) n_chunk * n_embd_enc);
                 for (uint32_t k = 0; k < target_layer_ids_n; ++k) {
                     const float * layer = llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]);
                     if (!layer) {
                         GGML_ABORT("DFlash: target layer %d input not extracted.", target_layer_ids[k]);
                     }
                     for (int32_t i = 0; i < n_chunk; ++i) {
-                        float       * dst = batch_inject.embd + (size_t) i * n_embd_enc + k * (size_t) n_embd_tgt;
+                        float       * dst = features_buf.data() + (size_t) i * n_embd_enc + k * (size_t) n_embd_tgt;
                         const float * src = layer + (size_t) (i_batch_beg[seq_id] + offset + i) * n_embd_tgt;
                         std::memcpy(dst, src, (size_t) n_embd_tgt * sizeof(float));
                     }
                 }
 
+                // fuse extracted features through DFlash encoder
+                llama_batch enc_batch = {
+                    /*.n_tokens =*/ n_chunk,
+                    /*.token    =*/ nullptr,
+                    /*.embd     =*/ features_buf.data(),
+                    /*.pos      =*/ nullptr,
+                    /*.n_seq_id =*/ nullptr,
+                    /*.seq_id   =*/ nullptr,
+                    /*.logits   =*/ nullptr,
+                };
+
+                int32_t rc = llama_encode(ctx_dft, enc_batch);
+                if (rc != 0) {
+                    LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
+                            __func__, rc, (int) n_chunk, (int) offset);
+                    return false;
+                }
+
+                const float * inp_g = llama_get_embeddings_nextn(ctx_dft);
+                GGML_ASSERT(inp_g && "DFlash encoder produced no output.");
+
+                // inject the DFlash decoder K/V cache at the tokens' target positions
+                batch_inject.n_tokens = n_chunk;
+                std::memcpy(batch_inject.embd, inp_g, (size_t) n_chunk * n_embd_dec * sizeof(float));
+
                 for (int32_t i = 0; i < n_chunk; ++i) {
-                    const llama_pos p = batch_in.pos[i_batch_beg[seq_id] + offset + i];
-                    batch_inject.pos[i] = p;
-                    if (is_mrope) {
-                        batch_inject.pos[1 * n_chunk + i] = p;
-                        batch_inject.pos[2 * n_chunk + i] = p;
-                        batch_inject.pos[3 * n_chunk + i] = 0;
-                    }
+                    batch_inject.pos[i]       = batch_in.pos[i_batch_beg[seq_id] + offset + i];
                     batch_inject.n_seq_id[i]  = 1;
                     batch_inject.seq_id[i][0] = seq_id;
                     batch_inject.logits[i]    = false;
                 }
-                const int32_t rc = llama_decode(ctx_dft, batch_inject);
+                rc = llama_decode(ctx_dft, batch_inject);
                 if (rc != 0) {
                     LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
                             __func__, rc, (int) n_chunk, (int) offset);
@@ -1197,7 +1178,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
             common_sampler_reset(smpls[seq_id].get());
 
-            const int32_t n = (int32_t) dp.pos0;
+            const int32_t n = (int32_t) dp.n_past;
 
             const int32_t n_draft = params.n_max;
 
@@ -1205,7 +1186,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             i_block_beg[seq_id] = batch.n_tokens;
             n_block    [seq_id] = n_block_tokens;
             for (int32_t i = 0; i < n_block_tokens; ++i) {
-                common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, !is_dflash2);
+                common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, true);
             }
         }
 
@@ -1233,36 +1214,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
             auto & result = *dp.result;
 
-            if (is_dflash2) {
-                const float * lattice = llama_get_embeddings_nextn(ctx_dft);
-                GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");
-
-                int32_t predecessor = 0;
-                for (int32_t i = 1; i < n_block_tokens; ++i) {
-                    const float * row = lattice + (size_t) (beg + i) * n_embd_dec;
-                    const float * scores = row + selector_top_k + (size_t) predecessor * selector_top_k;
-
-                    predecessor = (int32_t) std::distance(scores,
-                            std::max_element(scores, scores + selector_top_k));
-                    if (params.p_min > 0.0f) {
-                        // softmax(scores) at the argmax, i.e. 1 / sum(exp(s_k - s_max))
-                        float sum = 0.0f;
-                        for (int32_t k = 0; k < selector_top_k; ++k) {
-                            sum += std::exp(scores[k] - scores[predecessor]);
-                        }
-                        if (1.0f / sum < params.p_min) {
-                            break;
-                        }
-                    }
-                    result.push_back((llama_token) row[predecessor]);
-                }
-
-                if (result.size() < (size_t) params.n_min) {
-                    result.clear();
-                }
-                continue;
-            }
-
             if (is_dspark) {
                 // DSpark: read from the first draft slot, truncate below the confidence threshold
                 const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr;
@@ -1364,7 +1315,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
     std::vector<std::vector<float>> chain_h;
 
     common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq)
         , params(params.draft)
     {
         auto * ctx_tgt = this->params.ctx_tgt;
@@ -1431,7 +1382,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                 c.reserve((size_t) (this->params.n_max + 1) * n_embd);
             }
         }
-        this->n_max = this->params.n_max;
 
         pending_h.assign(n_seq, std::vector<float>(n_embd, 0.0f));
 
@@ -1493,7 +1443,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
 
         const int32_t n_tokens = batch_in.n_tokens;
 
-        // remember the first and last batch index for each sequence
+        // remember the frist and last batch index for each sequence
         std::fill(i_batch_beg.begin(), i_batch_beg.end(), -1);
         std::fill(i_batch_end.begin(), i_batch_end.end(), -1);
 
@@ -1621,7 +1571,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             drafting[seq_id] = true;
             common_sampler_reset(smpls[seq_id].get());
 
-            common_batch_add(batch, dp.id_last, dp.pos0, { seq_id }, true);
+            common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true);
             std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes);
 
             i_last[seq_id] = batch.n_tokens - 1;
@@ -1635,16 +1585,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
 
         while (n_drafting > 0) {
             // each step decodes under a different head, i.e. a different decoder layer, and
-            // KV is per layer. process() filled this layer's KV only for positions < pos0
+            // KV is per layer. process() filled this layer's KV only for positions < n_past
             // (prompt + accepted prefix) — nothing in the draft region yet. so reset the
-            // draft region (the seq_rm lower bound is pos0, leaving the prompt KV intact)
+            // draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact)
             // and select head i so it rebuilds its own layer's KV there; decoding just the
             // latest token would leave its attention reading cells only another head wrote.
             if (chain_heads) {
                 auto * mem_dft = llama_get_memory(ctx_dft);
                 for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) {
                     if (drafting[seq_id]) {
-                        llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].pos0, -1);
+                        llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1);
                     }
                 }
                 llama_set_nextn_layer_offset(ctx_dft, i);
@@ -1710,17 +1660,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                     const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far
                     for (int t = 0; t < n_rows; ++t) {
                         const llama_token tok = (t == 0) ? dp.id_last : result[t - 1];
-                        common_batch_add(batch, tok, dp.pos0 + t, { seq_id }, t == n_rows - 1);
+                        common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1);
                         std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd,
                                     chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes);
                     }
                 } else if (is_mem_shared) {
                     // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens
                     // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37
-                    common_batch_add(batch, id, dp.pos0, { seq_id }, true);
+                    common_batch_add(batch, id, dp.n_past, { seq_id }, true);
                     std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes);
                 } else {
-                    common_batch_add(batch, id, dp.pos0 + i + 1, { seq_id }, true);
+                    common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true);
                     std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes);
                 }
 
@@ -1776,7 +1726,7 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl {
     common_speculative_impl_ngram_simple(
             const common_params_speculative & params, uint32_t n_seq,
             common_ngram_simple_config config)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq, params.ngram_simple.size_m)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq)
         , params(params.ngram_simple)
         , config(config)
     {
@@ -1820,7 +1770,7 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
             const common_ngram_map & config,
             uint32_t n_seq)
         : common_speculative_impl(config.key_only ? COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K
-            : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq, config.size_value)
+            : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq)
     {
         for (uint32_t i = 0; i < n_seq; i++) {
             this->config.push_back(config);
@@ -1891,7 +1841,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl {
     common_speculative_impl_ngram_mod(
             const common_params_speculative & params,
             uint32_t n_seq)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq, params.ngram_mod.n_max)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq)
         , params(params.ngram_mod)
         , mod(params.ngram_mod.n_match, 4*1024*1024)
         , verbose(std::getenv("LLAMA_TRACE") != nullptr) {
@@ -2067,7 +2017,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {
             const std::string & path_dynamic,
             bool save_dynamic,
             bool save_static)
-        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq, n_draft)
+        : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq)
         , params(params.ngram_cache)
         , n_draft(n_draft)
         , save_dynamic(save_dynamic)
@@ -2188,8 +2138,6 @@ struct common_speculative {
 
     // which implementaion was used for a given seq_id
     std::vector<common_speculative_impl *> impl_last;
-
-    std::vector<double> synth_probs;
 };
 
 static common_ngram_map get_common_ngram_map(
@@ -2368,101 +2316,6 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {
     return n_max;
 }
 
-int32_t common_speculative_n_max(const common_speculative * spec) {
-    int32_t n_max = 0;
-
-    if (spec == nullptr) {
-        return n_max;
-    }
-
-    for (const auto & impl : spec->impls) {
-        n_max = std::max(n_max, std::max(0, impl->n_max));
-    }
-
-    return n_max;
-}
-
-std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max) {
-    const bool has_length = spec->synth_len != -1.0;
-    const bool has_rates  = !spec->synth_rates.empty();
-
-    if (!has_length && !has_rates) {
-        return {};
-    }
-    if (has_length && has_rates) {
-        throw std::invalid_argument("synthetic acceptance length and rates are mutually exclusive");
-    }
-
-    if (n_max <= 0) {
-        throw std::invalid_argument("synthetic acceptance requires at least one speculative token");
-    }
-
-    if (has_rates) {
-        const auto & rates = spec->synth_rates;
-        if (rates.size() != (size_t) n_max) {
-            throw std::invalid_argument(string_format(
-                    "synthetic acceptance rates must contain %d values, got %zu", n_max, rates.size()));
-        }
-
-        for (size_t i = 0; i < rates.size(); ++i) {
-            if (!std::isfinite(rates[i]) || rates[i] < 0.0 || rates[i] > 1.0) {
-                throw std::invalid_argument("synthetic acceptance rates must be finite and within [0, 1]");
-            }
-            if (i > 0 && rates[i] > rates[i - 1]) {
-                throw std::invalid_argument("synthetic acceptance rates must be monotonically non-increasing");
-            }
-        }
-
-        return rates;
-    }
-
-    const double length = spec->synth_len;
-    const double length_max = (double) n_max + 1.0;
-    if (!std::isfinite(length) || length < 1.0 || length > length_max) {
-        throw std::invalid_argument(string_format(
-                "synthetic acceptance length must be finite and within [1, %.0f]", length_max));
-    }
-
-    double p = 0.0;
-    if (length == length_max) {
-        p = 1.0;
-    } else if (length > 1.0) {
-        double p_min = 0.0;
-        double p_max = 1.0;
-        for (int i = 0; i < 32; ++i) {
-            const double p_mid = 0.5 * (p_min + p_max);
-            double sum = 0.0;
-            double term = p_mid;
-            for (int32_t j = 0; j < n_max; ++j) {
-                sum += term;
-                term *= p_mid;
-            }
-
-            if (sum < length - 1.0) {
-                p_min = p_mid;
-            } else {
-                p_max = p_mid;
-            }
-        }
-        p = 0.5 * (p_min + p_max);
-    }
-
-    std::vector<double> rates;
-    rates.reserve(n_max);
-    double rate = p;
-    for (int32_t i = 0; i < n_max; ++i) {
-        rates.push_back(rate);
-        rate *= p;
-    }
-
-    return rates;
-}
-
-const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec) {
-    GGML_ASSERT(spec);
-    return spec->synth_probs;
-}
-
 common_params common_base_params_to_speculative(const common_params & params) {
     const bool has_draft = params.speculative.has_dft();
 
@@ -2473,22 +2326,11 @@ common_params common_base_params_to_speculative(const common_params & params) {
     result.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED;
 
     if (has_draft) {
-        // default to global devices value
-        if (!params_spec.devices.empty()) {
-            result.devices           = params_spec.devices;
-        }
+        result.devices               = params_spec.devices;
         result.model                 = params_spec.mparams;
         result.n_gpu_layers          = params_spec.n_gpu_layers;
         result.tensor_buft_overrides = params_spec.tensor_buft_overrides;
 
-        // a draft pinned to a single device doesn't need the meta wrapper an inherited -sm tensor would give it
-        // (the device list is null-terminated, so a single device means size 2)
-        const size_t n_devs = std::count_if(params_spec.devices.begin(), params_spec.devices.end(),
-                [](ggml_backend_dev_t d) { return d != nullptr; });
-        if (n_devs == 1) {
-            result.split_mode = LLAMA_SPLIT_MODE_LAYER;
-        }
-
         if (params_spec.cpuparams.n_threads > 0) {
             result.cpuparams.n_threads       = params_spec.cpuparams.n_threads;
             result.cpuparams_batch.n_threads = params_spec.cpuparams_batch.n_threads;
@@ -2726,39 +2568,13 @@ common_speculative * common_speculative_init(common_params_speculative & params,
         return nullptr;
     }
 
-    common_speculative_ptr result(new common_speculative {
-        /* .dparams     = */ common_speculative_draft_params_vec(n_seq),
-        /* .impls       = */ std::move(impls),
-        /* .impl_last   = */ std::vector<common_speculative_impl *>(n_seq, nullptr),
-        /* .synth_probs = */ {},
-    });
+    auto * result = new common_speculative {
+        /* .dparams   = */ common_speculative_draft_params_vec(n_seq),
+        /* .impls     = */ std::move(impls),
+        /* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr)
+    };
 
-    const int32_t n_max_configured = common_speculative_n_max(&params);
-    const int32_t n_max_effective  = common_speculative_n_max(result.get());
-    const auto rates = common_speculative_synth_rates_resolve(&params, n_max_effective);
-
-    std::vector<std::string> rates_str;
-    rates_str.reserve(rates.size());
-    result->synth_probs.reserve(rates.size());
-    double rate_prev = 1.0;
-    double acceptance_length = 1.0;
-    for (const double rate : rates) {
-        result->synth_probs.push_back(rate_prev > 0.0 ? rate / rate_prev : 0.0);
-        rates_str.push_back(string_format("%.6g", rate));
-        rate_prev = rate;
-        acceptance_length += rate;
-    }
-    if (!result->synth_probs.empty()) {
-        SPC_WRN("%s", "synthetic speculative acceptance is enabled for benchmarking; generated output is not valid\n");
-        if (n_max_effective != n_max_configured) {
-            SPC_WRN("synthetic acceptance draft limit was reduced from %d to %d by the initialized speculative implementations\n",
-                    n_max_configured, n_max_effective);
-        }
-        SPC_INF("synthetic acceptance: n_max = %zu, mean length = %.6f, rates = [%s]\n",
-                rates.size(), acceptance_length, string_join(rates_str, ", ").c_str());
-    }
-
-    return result.release();
+    return result;
 }
 
 void common_speculative_free(common_speculative * spec) {

common/speculative.h

@@ -26,15 +26,6 @@ std::string common_speculative_type_to_str(enum common_speculative_type type);
 // return the max number of draft tokens based on the speculative parameters
 int32_t common_speculative_n_max(const common_params_speculative * spec);
 
-// return the max number of draft tokens from the initialized implementations
-int32_t common_speculative_n_max(const common_speculative * spec);
-
-// validate and resolve the unconditional synthetic acceptance rates
-std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max);
-
-// return the conditional synthetic acceptance probabilities
-const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec);
-
 common_params common_base_params_to_speculative(const common_params & params);
 
 struct common_speculative_output_limits {
@@ -61,7 +52,7 @@ struct common_speculative_draft_params {
     // can be used to constraint the max draft based on the remaining context size
     int32_t n_max = -1;
 
-    llama_pos   pos0;
+    llama_pos   n_past;
     llama_token id_last;
 
     // TODO: remove in the future by keeping track of the prompt from the _begin() call and the consecutive accept calls

conversion/__init__.py

@@ -54,7 +54,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "DeepseekV3ForCausalLM": "deepseek",
     "DeepseekV32ForCausalLM": "deepseek",
     "DFlashDraftModel": "qwen",
-    "DFlash2DraftModel": "qwen",
     "Qwen3DSparkModel": "qwen",
     "DSparkDraftModel": "qwen",
     "DSparkSpeculator": "qwen",
@@ -124,7 +123,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "HunYuanMoEV1ForCausalLM": "hunyuan",
     "HunYuanVLForConditionalGeneration": "hunyuan",
     "HYV3ForCausalLM": "hunyuan",
-    "HYV4ForCausalLM": "hy_v4",
     "IQuestCoderForCausalLM": "llama",
     "InternLM2ForCausalLM": "internlm",
     "InternLM3ForCausalLM": "internlm",
@@ -189,7 +187,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "NanbeigeForCausalLM": "nanbeige",
     "NemotronForCausalLM": "nemotron",
     "NemotronHForCausalLM": "nemotron",
-    "NemotronHPuzzleForCausalLM": "nemotron",
     "NeoBERT": "bert",
     "NeoBERTForSequenceClassification": "bert",
     "NeoBERTLMHead": "bert",
@@ -238,8 +235,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "Qwen3_5ForConditionalGeneration": "qwen",
     "Qwen3_5MoeForCausalLM": "qwen",
     "Qwen3_5MoeForConditionalGeneration": "qwen",
-    "Qwen4ExpForCausalLM": "qwen4exp",
-    "Qwen4ExpForConditionalGeneration": "qwen4exp",
     "RND1": "qwen",
     "RWForCausalLM": "falcon",
     "RWKV6Qwen2ForCausalLM": "rwkv",
@@ -255,7 +250,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "SeedOssForCausalLM": "olmo",
     "SmallThinkerForCausalLM": "smallthinker",
     "SmolLM3ForCausalLM": "llama",
-    "Spark2_5ForCausalLM": "spark2_5",
     "SolarOpenForCausalLM": "glm",
     "StableLMEpochForCausalLM": "stablelm",
     "StableLmForCausalLM": "stablelm",
@@ -289,7 +283,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "CogVLMForCausalLM": "cogvlm",
     "DeepseekOCR2ForCausalLM": "deepseek",
     "DeepseekOCRForCausalLM": "deepseek",
-    "DeepseekV4ForCausalLM": "deepseek",
     "Dots3NoteForCausalLM": "dots3",
     "Dots3NoteForConditionalGeneration": "dots3",
     "DotsOCRForCausalLM": "dotsocr",
@@ -339,7 +332,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Qwen3VLMoeForConditionalGeneration": "qwen3vl",
     "Qwen3_5ForConditionalGeneration": "qwen3vl",
     "Qwen3_5MoeForConditionalGeneration": "qwen3vl",
-    "Qwen4ExpForConditionalGeneration": "qwen4exp",
     "RADIOModel": "nemotron",
     "Sarashina2VisionForCausalLM": "sarashina2",
     "SmolVLMForConditionalGeneration": "smolvlm",

conversion/base.py

@@ -130,8 +130,7 @@ class ModelBase:
                  sentence_transformers_dense_modules: bool = False,
                  target_model_dir: Path | None = None,
                  fuse_gate_up_exps: bool = False,
-                 fp8_as_q8: bool = False,
-                 fuse_qkv: bool = False):
+                 fp8_as_q8: bool = False):
         if type(self) is ModelBase or \
                 type(self) is TextModel or \
                 type(self) is MmprojModel:
@@ -154,15 +153,6 @@ class ModelBase:
         self.fuse_gate_up_exps = fuse_gate_up_exps
         self._gate_exp_buffer: dict[int, Tensor] = {}
         self._up_exp_buffer: dict[int, Tensor] = {}
-        self.fuse_qkv = fuse_qkv
-        self._q_buffer: dict[int, Tensor] = {}
-        self._k_buffer: dict[int, Tensor] = {}
-        self._v_buffer: dict[int, Tensor] = {}
-        self._q_bias_buffer: dict[int, Tensor] = {}
-        self._k_bias_buffer: dict[int, Tensor] = {}
-        self._v_bias_buffer: dict[int, Tensor] = {}
-        self._fusable_qkv_weight_layers: set[int] = set()
-        self._fusable_qkv_bias_layers: set[int] = set()
         self.hparams = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) if hparams is None else hparams
         self.model_tensors = self.index_tensors(remote_hf_model_id=remote_hf_model_id)
         self.metadata_override = metadata_override
@@ -627,43 +617,6 @@ class ModelBase:
             raise ValueError(f"Can not map tensor {name!r}")
         return new_name
 
-    def prepare_qkv_fusion(self) -> None:
-        self._fusable_qkv_weight_layers.clear()
-        self._fusable_qkv_bias_layers.clear()
-        if not self.fuse_qkv or gguf.MODEL_TENSOR.ATTN_QKV not in gguf.MODEL_TENSORS[self.model_arch]:
-            return
-
-        qkv_types = {
-            gguf.MODEL_TENSOR.ATTN_Q,
-            gguf.MODEL_TENSOR.ATTN_K,
-            gguf.MODEL_TENSOR.ATTN_V,
-        }
-        weights: dict[int, set[gguf.MODEL_TENSOR]] = {}
-        biases: dict[int, set[gguf.MODEL_TENSOR]] = {}
-
-        for name in self.model_tensors:
-            mapped = self.tensor_map.get_type_and_name(name, try_suffixes=(".weight", ".bias"))
-            if mapped is None:
-                continue
-            tensor_type, new_name = mapped
-            if tensor_type not in qkv_types:
-                continue
-
-            bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None)
-            if bid is None:
-                continue
-            if new_name.endswith(".weight"):
-                weights.setdefault(bid, set()).add(tensor_type)
-            elif new_name.endswith(".bias"):
-                biases.setdefault(bid, set()).add(tensor_type)
-
-        for bid, weight_types in weights.items():
-            bias_types = biases.get(bid, set())
-            if weight_types == qkv_types and (not bias_types or bias_types == qkv_types):
-                self._fusable_qkv_weight_layers.add(bid)
-                if bias_types:
-                    self._fusable_qkv_bias_layers.add(bid)
-
     def set_gguf_parameters(self):
         raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses")
 
@@ -692,40 +645,6 @@ class ModelBase:
                self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_UP_EXP, bid):
                 return []
 
-        # Handle Q/K/V tensor fusion if enabled
-        qkv_bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None) if self.fuse_qkv else None
-        if qkv_bid is not None:
-            is_bias = new_name.endswith('.bias')
-            suffix = '.bias' if is_bias else '.weight'
-            fusable_layers = self._fusable_qkv_bias_layers if is_bias else self._fusable_qkv_weight_layers
-            if qkv_bid not in fusable_layers:
-                return [(new_name, data_torch)]
-
-            buf_q = self._q_bias_buffer if is_bias else self._q_buffer
-            buf_k = self._k_bias_buffer if is_bias else self._k_buffer
-            buf_v = self._v_bias_buffer if is_bias else self._v_buffer
-
-            if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_Q, qkv_bid, suffix):
-                buf_q[qkv_bid] = data_torch
-            elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_K, qkv_bid, suffix):
-                buf_k[qkv_bid] = data_torch
-            elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_V, qkv_bid, suffix):
-                buf_v[qkv_bid] = data_torch
-
-            if qkv_bid in buf_q and qkv_bid in buf_k and qkv_bid in buf_v:
-                q_data = buf_q.pop(qkv_bid)
-                k_data = buf_k.pop(qkv_bid)
-                v_data = buf_v.pop(qkv_bid)
-                fused_data = torch.cat([q_data, k_data, v_data], dim=0)
-                fused_name = self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, qkv_bid, suffix=suffix)
-                logger.info(f"Fused Q, K, V {suffix[1:]} into QKV for layer {qkv_bid}")
-                return [(fused_name, fused_data)]
-
-            if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_Q, qkv_bid, suffix) or \
-               self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_K, qkv_bid, suffix) or \
-               self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.ATTN_V, qkv_bid, suffix):
-                return []
-
         return [(new_name, data_torch)]
 
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
@@ -980,8 +899,6 @@ class ModelBase:
 
         self.dequant_model()
 
-        self.prepare_qkv_fusion()
-
         # Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
         if self.tensor_map.mapping:
             max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
@@ -1089,16 +1006,12 @@ class ModelBase:
                     else:
                         raise ValueError(f"Unknown file type: {self.ftype.name}")
 
-                # a chunked tensor quantizes as one chunk at a time, while it is written
-                quantize = data.quantize if isinstance(data, gguf.LazyChunkedTensor) else (
-                    lambda qtype, d=data: gguf.quants.quantize(d, qtype))
-
                 try:
-                    data = quantize(data_qtype)
+                    data = gguf.quants.quantize(data, data_qtype)
                 except gguf.QuantError as e:
                     logger.warning("%s, %s", e, "falling back to F16")
                     data_qtype = gguf.GGMLQuantizationType.F16
-                    data = quantize(data_qtype)
+                    data = gguf.quants.quantize(data, data_qtype)
 
                 shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
 
@@ -1110,13 +1023,6 @@ class ModelBase:
 
                 self.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)
 
-        qkv_buffers = (
-            self._q_buffer, self._k_buffer, self._v_buffer,
-            self._q_bias_buffer, self._k_bias_buffer, self._v_bias_buffer,
-        )
-        if any(qkv_buffers):
-            raise ValueError("QKV fusion did not consume all buffered tensors")
-
     def set_type(self):
         self.gguf_writer.add_type(gguf.GGUFType.MODEL)
 
@@ -1597,9 +1503,6 @@ class TextModel(ModelBase):
         if chkhsh == "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6":
             # ref: https://huggingface.co/tencent/Hunyuan-4B-Instruct
             res = "hunyuan-dense"
-        if chkhsh == "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c":
-            # ref: https://huggingface.co/tencent/Hy4-preview
-            res = "hy_v4"
         if chkhsh == "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6":
             # ref: https://huggingface.co/tiiuae/Falcon-H1-0.5B-Base
             res = "falcon-h1"
@@ -1633,9 +1536,6 @@ class TextModel(ModelBase):
         if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
             # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
             res = "lfm2"
-        if chkhsh == "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed":
-            # ref: https://huggingface.co/XHToken/Spark-X2.5-1.7B
-            res = "spark2_5"
         if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
             # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
             res = "llama-bpe"

conversion/deepseek.py

@@ -578,8 +578,6 @@ class DeepseekV4Model(TextModel):
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         name, gen = item
-        if name.startswith(("aligner.", "image_")):
-            return None
         if name.startswith("mtp."):
             if not cls.mtp_only:
                 cls._skipped_mtp_tensors += 1
@@ -855,7 +853,6 @@ class DeepseekV4Model(TextModel):
             "ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"),
             "ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"),
             "ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"),
-            "ffn.gate.bias_vl": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B_VL, ".bias"),
             "ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"),
             "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"),
             "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"),
@@ -881,10 +878,6 @@ class DeepseekV4Model(TextModel):
         if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name):
             return []
 
-        # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused
-        if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]:
-            return []
-
         tensor_key, suffix = self._map_dsv4_tensor_name(name, bid)
         if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID:
             return []
@@ -1007,13 +1000,6 @@ class DeepseekV4DSparkModel(DeepseekV4Model):
             return self._DSPARK_ROOT_MAP[name]
         return super()._map_dsv4_tensor_name(name, bid)
 
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # the DFlash draft uses the plain exp-probs bias (ffn.gate.bias -> FFN_EXP_PROBS_B);
-        # the mtmd-only hash routing tensors (bias_vl, tid2eid) are not part of the DFLASH arch
-        if name.endswith(".ffn.gate.bias_vl"):
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
     def set_vocab(self):
         if self.target_model_dir is None:
             raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer")
@@ -1032,73 +1018,3 @@ class DeepseekV4DSparkModel(DeepseekV4Model):
 
         self.gguf_writer.add_block_size(self.hparams["dspark_block_size"])
         self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]])
-
-
-@ModelBase.register("DeepseekV4ForCausalLM")
-@ModelBase.example("deepseek-ai/DeepSeek-V4-Flash-Vision-Exp")
-class DeepseekV4FlashVisionModel(MmprojModel):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # no preprocessor_config.json in the repo; normalization is (x/255 - 0.5) / 0.5
-        # ref: inference/image_processor.py (load_image)
-        self.preprocessor_config = {
-            "image_mean": [0.5, 0.5, 0.5],
-            "image_std":  [0.5, 0.5, 0.5],
-            **self.preprocessor_config,
-        }
-
-    def get_vision_config(self) -> dict[str, Any] | None:
-        cfg = self.global_config
-        if cfg.get("vision_n_layers", 0) == 0:
-            raise ValueError("DeepseekV4FlashVisionModel requires vision_n_layers > 0 in the model config")
-        return {
-            "num_hidden_layers":   cfg["vision_n_layers"],
-            "hidden_size":         cfg["vision_dim"],
-            "num_attention_heads": cfg["vision_n_heads"],
-            "intermediate_size":   cfg["vision_inter_dim"],
-            "patch_size":          cfg["vision_patch_size"],
-            # dynamic resolution; only used for compat / warmup
-            "image_size":          cfg["vision_patch_size"] * cfg["vision_downsample_ratio"] * 16,
-            "rope_theta":          cfg.get("vision_rope_theta", 10000.0),
-            "downsample_ratio":    cfg["vision_downsample_ratio"],
-            "min_pixels":          cfg["vision_min_pixels"],
-        }
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        assert self.hparams_vision is not None
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK4V)
-        # vision RMSNorm eps is the pytorch default, NOT the LLM's rms_norm_eps (1e-20)
-        # ref: inference/vision.py (RMSNorm)
-        self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
-        self.gguf_writer.add_vision_use_silu(True) # SwiGLU MLP
-        self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
-        self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
-        # hardcoded on the C++ side (see PROJECTOR_TYPE_DEEPSEEK4V in clip.cpp)
-        # if future models use different values, add GGUF keys for those
-        assert self.global_config["vision_max_n_token"] == 384
-        assert self.global_config["vision_max_wh_ratio"] == 8
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        if not (name.startswith(("vision.", "aligner.", "image_"))):
-            return None
-        return super().filter_tensors(item)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        assert self.hparams_vision is not None
-        if name == "vision.patch_embed.proj.weight":
-            # nn.Linear over flattened (3, p, p) patches == conv2d weight
-            p = self.hparams_vision["patch_size"]
-            data_torch = data_torch.reshape(data_torch.shape[0], 3, p, p)
-
-        if ".mlp.w1." in name:
-            # fused SwiGLU gate+up
-            gate, up = data_torch.chunk(2, dim=0)
-            yield from super().modify_tensors(gate, name.replace("w1", "w1_gate"), bid)
-            yield from super().modify_tensors(up, name.replace("w1", "w1_up"), bid)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/glm.py

@@ -112,38 +112,12 @@ class GlmOCRModel(Glm4Model):
 @ModelBase.example("zai-org/GLM-4.5-Air")
 class Glm4MoeModel(TextModel):
     model_arch = gguf.MODEL_ARCH.GLM4_MOE
-    supports_mtp_export = True
-    _n_main_layers: int | None = None
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        if not self.no_mtp:
-            self.block_count += self.hparams.get("num_nextn_predict_layers", 0)
-            self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None):
-        hparams = {**self.hparams, **self.hparams.get("text_config", {})}
-        key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None)
-        type(self)._n_main_layers = hparams.get(key)
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        if (titem := super().filter_tensors(item)) is None:
-            return None
-        name, gen = titem
-
-        assert cls._n_main_layers is not None
-        is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers
-
-        if is_mtp and cls.no_mtp:
-            return None
-        if cls.mtp_only and not is_mtp and name not in (
-            "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
-        ):
-            return None
-
-        return name, gen
+        # GLM4_MOE has num_hidden_layers + 1 actual layers (including NextN layer)
+        self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
+        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
 
     def set_vocab(self):
         return self._set_vocab_glm()
@@ -179,22 +153,10 @@ class Glm4MoeModel(TextModel):
         if (norm_topk_prob := self.hparams.get("norm_topk_prob")) is not None:
             self.gguf_writer.add_expert_weights_norm(norm_topk_prob)
 
-        if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
+        # NextN/MTP prediction layers
+        if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
             self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
 
-    def prepare_metadata(self, vocab_only: bool):
-        from_dir = self.fname_out.is_dir()
-        super().prepare_metadata(vocab_only=vocab_only)
-
-        if not self.mtp_only or not from_dir:
-            return
-
-        output_type: str = self.ftype.name.partition("_")[2]
-        fname_default: str = gguf.naming_convention(
-            self.metadata.name, self.metadata.basename, self.metadata.finetune,
-            self.metadata.version, size_label=None, output_type=output_type, model_type=None)
-        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
-
     _experts: list[dict[str, Tensor]] | None = None
 
     # note: unlike GLM4V non-MoE, we don't need to permute Q/K here since GLM4V_MOE uses Neox ordering already
@@ -386,7 +348,6 @@ class GlmMoeDsaModel(DeepseekV2Model):
 @ModelBase.example("upstage/Solar-Open-100B")
 class SolarOpenModel(Glm4MoeModel):
     model_arch = gguf.MODEL_ARCH.GLM4_MOE
-    supports_mtp_export = False
 
     def set_vocab(self):
         from transformers import AutoTokenizer

conversion/hy_v4.py

@@ -1,244 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Iterable
-
-import torch
-
-from .base import ModelBase, gguf, logger
-from .deepseek import DeepseekV2Model
-
-
-def split_gate_up(weight: torch.Tensor, moe_intermediate_size: int):
-    """Split a fused stacked gate_up expert tensor into (gate, up).
-
-    weight: [n_expert, 2*moe_intermediate_size, hidden] (gate first, up second).
-    Returns (gate, up) each [n_expert, moe_intermediate_size, hidden].
-    """
-    assert weight.shape[1] == 2 * moe_intermediate_size, f"{weight.shape[1]} != 2*{moe_intermediate_size}"
-    gate = weight[:, :moe_intermediate_size, :].contiguous()
-    up = weight[:, moe_intermediate_size:, :].contiguous()
-    return gate, up
-
-
-@ModelBase.register("HYV4ForCausalLM")
-@ModelBase.example("tencent/Hy4-preview")
-class HYV4Model(DeepseekV2Model):
-    """HY_V4: DeepSeek-V3 style MLA + MoE with iHC, a gated MLA output and a learnable sink.
-
-    Reuses DeepseekV2Model for the vocab and the MLA metadata, but overrides the tensor mapping
-    because HY_V4 ships pre-stacked / fused experts plus extra iHC, gate and sink tensors. The
-    rope rows are mapped straight through (no permute) - the graph rotates consecutive pairs.
-
-    DSA is supported: indexer weights are exported for the layers marked "full" in indexer_types.
-    "shared" layers reuse the top-k of the last preceding full layer at inference time, so they
-    carry no indexer weights.
-
-    MTP (num_nextn_predict_layers) is dropped, so the GGUF cannot be used for speculative
-    decoding. The reference only runs the MTP layers while training or while speculating, so they
-    cannot change single-token logits.
-    """
-
-    model_arch = gguf.MODEL_ARCH.HY_V4
-
-    merge_expert = False
-
-    # tensors a "full" indexer layer must carry
-    INDEXER_SUFFIXES = frozenset({
-        "self_attn.indexer.wq_b.weight",
-        "self_attn.indexer.wk.weight",
-        "self_attn.indexer.k_norm.weight",
-        "self_attn.indexer.k_norm.bias",
-        "self_attn.indexer.weights_proj.weight",
-    })
-
-    @classmethod
-    def filter_tensors(cls, item):
-        # drop MTP here, not in modify_tensors, so the weights are never read
-        if item[0].startswith("model.mtp_layers."):
-            return None
-        return super().filter_tensors(item)
-
-    def _check_indexer_hparams(self):
-        for key in ("index_n_heads", "index_head_dim", "index_topk"):
-            if key not in self.hparams:
-                raise ValueError(f"HY_V4 has DSA layers but no {key}")
-
-    def indexer_is_full(self) -> list[bool] | None:
-        """Per-layer indexer ownership, or None when the checkpoint has no DSA.
-
-        indexer_types entries are "full" (owns an indexer) or "shared" (reuses the preceding
-        full layer's top-k). Missing indexer_types with sparse layers means every sparse layer
-        owns one.
-        """
-        hparams = self.hparams
-        n_layer = hparams["num_hidden_layers"]
-        indexer_types = hparams.get("indexer_types")
-
-        # the reference drives DSA off indexer_types alone; layer_types is only a fallback for
-        # checkpoints predating it (it was renamed to deepseek_sparse_attention upstream)
-        if indexer_types is None:
-            layer_types = hparams.get("layer_types") or []
-            sparse = {"sparse_attention", "deepseek_sparse_attention"}
-            if not any(t in sparse for t in layer_types):
-                return None
-            if len(layer_types) < n_layer:
-                raise ValueError(f"HY_V4 layer_types has {len(layer_types)} entries, need {n_layer}")
-            self._check_indexer_hparams()
-            return [t in sparse for t in layer_types[:n_layer]]
-
-        self._check_indexer_hparams()
-
-        if len(indexer_types) < n_layer:
-            raise ValueError(f"HY_V4 indexer_types has {len(indexer_types)} entries, need {n_layer}")
-        unknown = {t for t in indexer_types[:n_layer]} - {"full", "shared"}
-        if unknown:
-            raise ValueError(f"HY_V4 unknown indexer_types values: {sorted(unknown)}")
-        is_full = [t == "full" for t in indexer_types[:n_layer]]
-        if is_full and not is_full[0]:
-            raise ValueError("HY_V4 layer 0 must be indexer_types 'full' (nothing precedes it to share)")
-        return is_full
-
-    def set_gguf_parameters(self):
-        hparams = self.hparams
-
-        # HY4 has n_group == topk_group == 1 (no group routing). Drop the keys so the base does
-        # not emit expert_group_count/used; llama.cpp then takes the ungrouped MoE path.
-        if hparams.get("n_group") == 1 and hparams.get("topk_group") == 1:
-            hparams.pop("n_group", None)
-            hparams.pop("topk_group", None)
-
-        # HY_V4 config expresses dense/sparse layers via mlp_layer_types, but DeepseekV2Model
-        # needs first_k_dense_replace. Derive it as the contiguous leading "dense" block
-        # (the real config.json also carries first_k_dense_replace; prefer it when present,
-        # but assert the two agree so a mismatch fails loudly).
-        mlp_types = hparams.get("mlp_layer_types")
-        explicit = hparams.get("first_k_dense_replace")
-        derived = None
-        if mlp_types is not None:
-            lead = 0
-            for t in mlp_types:
-                if t == "dense":
-                    lead += 1
-                else:
-                    break
-            if any(t == "dense" for t in mlp_types[lead:]):
-                raise NotImplementedError("HY_V4 converter expects a contiguous leading dense block")
-            derived = lead
-        if explicit is not None and derived is not None and explicit != derived:
-            raise ValueError(
-                f"HY_V4 first_k_dense_replace ({explicit}) disagrees with mlp_layer_types "
-                f"leading-dense count ({derived})"
-            )
-        if explicit is None:
-            if derived is None:
-                raise ValueError("HY_V4 needs first_k_dense_replace or mlp_layer_types to place dense layers")
-            hparams["first_k_dense_replace"] = derived
-
-        # reuse DeepseekV2 MLA + MoE metadata (forces num_key_value_heads=1, writes q/kv lora,
-        # key/value lengths, expert counts, weights scale/norm, rope dims, etc.)
-        super().set_gguf_parameters()
-
-        # HY4 uses DeepSeek-V3 sigmoid routing with e_score_correction_bias. The config has no
-        # scoring_func key, so the base does not write a gating func; set it explicitly.
-        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
-
-        # routed-expert SwiGLU logits clamp (only routed experts; shared/dense are not clamped,
-        # so swiglu_clamp_shexp is intentionally not written). 0.0 disables the clamp.
-        swiglu_limit = float(hparams.get("swiglu_limit", 0.0) or 0.0)
-        if swiglu_limit > 0.0:
-            self.gguf_writer.add_swiglu_clamp_exp([swiglu_limit] * self.block_count)
-
-        # iHC (independent Hyper-Connections)
-        self.gguf_writer.add_hyper_connection_count(hparams["hc_mult"])
-        self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"])
-        self.gguf_writer.add_hyper_connection_magnitude(hparams["hc_magnitude"])
-
-        # is_full is written explicitly; the graph must not infer it from tensor presence
-        is_full = self.indexer_is_full()
-        if is_full is not None:
-            self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"])
-            self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"])
-            self.gguf_writer.add_indexer_top_k(hparams["index_topk"])
-            self.gguf_writer.add_indexer_types(is_full)
-            logger.info(
-                "HY_V4 DSA: %d/%d layers own an indexer (top_k=%d, n_heads=%d, head_dim=%d)",
-                sum(is_full), len(is_full), hparams["index_topk"],
-                hparams["index_n_heads"], hparams["index_head_dim"],
-            )
-
-        if hparams.get("num_nextn_predict_layers", 0):
-            logger.warning(
-                "HY_V4: dropping %d MTP (nextn) layer(s) - the reference runs them only under "
-                "training / speculative decoding. This GGUF cannot be used for speculative decoding.",
-                hparams["num_nextn_predict_layers"],
-            )
-
-    def prepare_tensors(self):
-        # Hy4-preview for some reason has num_key_value_heads equal to 8, so override it here
-        # without this conversion/deepseek.py fails on assert
-        self.hparams["num_key_value_heads"] = self.hparams["num_attention_heads"]
-
-        # validate before the base materializes tensors, so a mismatch fails early
-        is_full = self.indexer_is_full()
-        if is_full is not None:
-            present: dict[int, set[str]] = {}
-            for name in self.model_tensors:
-                m = re.match(r"model\.layers\.(\d+)\.(self_attn\.indexer\..+)$", name)
-                if m:
-                    present.setdefault(int(m.group(1)), set()).add(m.group(2))
-            for il, expect_full in enumerate(is_full):
-                seen = present.get(il, set())
-                if expect_full and seen != self.INDEXER_SUFFIXES:
-                    raise ValueError(
-                        f"HY_V4 layer {il} is indexer_types 'full' but is missing indexer tensors: "
-                        f"{sorted(self.INDEXER_SUFFIXES - seen)}"
-                    )
-                if not expect_full and seen:
-                    raise ValueError(
-                        f"HY_V4 layer {il} is indexer_types 'shared' but carries indexer tensors: "
-                        f"{sorted(seen)}"
-                    )
-
-        super().prepare_tensors()
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # iHC mixing matrices are 2D .weight tensors that the reference keeps in fp32
-        # (_keep_in_fp32_modules_strict). 1D tensors (hc_base/scale, attn_sinks,
-        # e_score_correction_bias) and the router (FFN_GATE_INP) are already forced F32 by the
-        # base rules. Force the HC *_fn matrices here.
-        if new_name.endswith(("hc_attn_fn.weight", "hc_ffn_fn.weight", "output_hc_fn.weight")):
-            return gguf.GGMLQuantizationType.F32
-        # indexer k_norm is fp32 in the reference; the base rules already cover
-        # *_norm.weight and INDEXER_PROJ, but not this bias
-        if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.INDEXER_K_NORM, bid, suffix=".bias"):
-            return gguf.GGMLQuantizationType.F32
-        # enable_lm_head_fp32: mirror the reference fp32 LM-head matmul by keeping output F32.
-        if new_name == "output.weight" and self.hparams.get("enable_lm_head_fp32", False):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def modify_tensors(self, data_torch: torch.Tensor, name: str, bid: int | None) -> Iterable[tuple[str, torch.Tensor]]:
-        hparams = self.hparams
-        moe_inter = hparams["moe_intermediate_size"]
-
-        tn = self.format_tensor_name
-
-        # fused stacked experts: split gate_up into gate/up
-        if name.endswith("mlp.experts.gate_up_proj"):
-            gate, up = split_gate_up(data_torch, moe_inter)
-            yield from super().modify_tensors(gate, tn(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid), bid)
-            yield from super().modify_tensors(up,   tn(gguf.MODEL_TENSOR.FFN_UP_EXP,   bid), bid)
-            return
-
-        # add .weight suffixes
-        if name.endswith("mlp.experts.down_proj") or name.endswith(".self_attn.learnable_sink_param"):
-            name += ".weight"
-
-        if re.search(r"\.hc_head\.hc_head_(?:fn|base|scale)$", name):
-            name += ".weight"
-
-        if re.search(r"\.hc_(?:attn|mlp)_layer\.hc_pre\.hc_(?:fn|base|scale)$", name):
-            name += ".weight"
-
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/minimax.py

@@ -25,7 +25,7 @@ class MiniMaxText01Model(TextModel):
         # they get in the way of the token sampling process and must be suppressed
 
         tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
-        tokenizer_vocab_size = tokenizer.vocab_size  # ty: ignore[unresolved-attribute]
+        tokenizer_vocab_size = tokenizer.vocab_size
 
         with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f:
             weight_map = json.load(f)["weight_map"]

conversion/muse_glimmer.py

@@ -37,7 +37,7 @@ class MuseGlimmerModel(TextModel):
 
         from transformers import AutoTokenizer
         tok = AutoTokenizer.from_pretrained(self.dir_model)
-        eot_id = tok.convert_tokens_to_ids("<|eot|>")  # ty: ignore[unresolved-attribute]
+        eot_id = tok.convert_tokens_to_ids("<|eot|>")
         if isinstance(eot_id, int) and eot_id >= 0:
             self.gguf_writer.add_eot_token_id(eot_id)
 

conversion/nemotron.py

@@ -5,7 +5,6 @@ from typing import Any, Callable, Iterable, TYPE_CHECKING
 import torch
 
 if TYPE_CHECKING:
-    from pathlib import Path
     from torch import Tensor
 
 from .base import MmprojModel, ModelBase, TextModel, gguf, logger
@@ -202,11 +201,6 @@ class NemotronHModel(GraniteHybridModel):
     model_arch = gguf.MODEL_ARCH.NEMOTRON_H
     is_moe: bool = False
     supports_mtp_export = True
-    _experts: list[dict[str, Tensor]] | None = None
-
-    _SSM_LAYER_TYPES = {"mamba", "linear_attention"}
-    _ATTN_LAYER_TYPES = {"attention", "full_attention"}
-    _MLP_LAYER_TYPES = {"moe"}
 
     def __init__(self, *args, **kwargs):
         # We have to determine the correct model architecture (MoE vs non-MoE) before
@@ -216,14 +210,14 @@ class NemotronHModel(GraniteHybridModel):
         hparams = kwargs.pop("hparams", None)
         if hparams is None:
             hparams = ModelBase.load_hparams(args[0], self.is_mistral_format)
-        llm_config = {**hparams, **(hparams.get("llm_config") or {})}
-
-        has_moe_params = "num_experts_per_tok" in llm_config
-        layers_block_type = llm_config.get("layers_block_type")
-
+        has_moe_params = (
+            "num_experts_per_tok" in hparams
+            or (isinstance(hparams.get("llm_config"), dict) and "num_experts_per_tok" in hparams["llm_config"])
+        )
         if has_moe_params:
             self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE
             self.is_moe = True
+        layers_block_type = hparams.get("layers_block_type")
         if layers_block_type is not None:
             hparams["num_hidden_layers"] = len(layers_block_type)
 
@@ -248,8 +242,8 @@ class NemotronHModel(GraniteHybridModel):
             self._ssm_layers = [i for i, val in enumerate(pattern) if val == "M"]
             self._mlp_layers = [i for i, val in enumerate(pattern) if val == ("E" if self.is_moe else "-")]
         else:
-            self._ssm_layers = [i for i, val in enumerate(pattern) if val in self._SSM_LAYER_TYPES]
-            self._mlp_layers = [i for i, val in enumerate(pattern) if val in self._MLP_LAYER_TYPES]
+            self._ssm_layers = [i for i, val in enumerate(pattern) if val == "mamba"]
+            self._mlp_layers = [i for i, val in enumerate(pattern) if val == "moe"]
 
         # `--no-mtp` drops it entirely; `--mtp` exports only the MTP head
         self._mtp_bid: int | None = None
@@ -278,7 +272,7 @@ class NemotronHModel(GraniteHybridModel):
         if isinstance(pattern, str):
             return [i for i, val in enumerate(pattern) if val == "*"]
 
-        return [i for i, val in enumerate(pattern) if val in self._ATTN_LAYER_TYPES]
+        return [i for i, val in enumerate(pattern) if val == "attention"]
 
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
@@ -304,10 +298,6 @@ class NemotronHModel(GraniteHybridModel):
             )
             if not keep:
                 return None
-        # PEFT names adapter tensors using model.layers.*, while Nemotron-H checkpoints
-        # and the GGUF tensor map use backbone.layers.*
-        if name.startswith("model.layers.") and ".mixer." in name:
-            name = name.replace("model.layers.", "backbone.layers.", 1)
         return super().filter_tensors((name, gen))
 
     def prepare_metadata(self, vocab_only: bool):
@@ -515,88 +505,3 @@ class NemotronHModel(GraniteHybridModel):
             experts = [k for d in self._experts for k in d.keys()]
             if len(experts) > 0:
                 raise ValueError(f"Unprocessed experts: {experts}")
-
-
-@ModelBase.register("NemotronHPuzzleForCausalLM")
-@ModelBase.example("nvidia/NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16")
-class NemotronHPuzzleModel(NemotronHModel):
-    """NVIDIA Puzzle: NemotronH with a per-block MoE config (block_configs).
-
-    The checkpoint also ships an MTP draft head (mtp.safetensors). It is skipped
-    here: there is no Puzzle MTP inference path in tree, and the head is laid out
-    by mtp_block_configs rather than the mtp.layers.* form NemotronHModel maps."""
-
-    model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE
-    is_moe: bool = True
-    supports_mtp_export = False
-
-    def __init__(self, dir_model: "Path", *args, **kwargs):
-        hparams = dict(kwargs.pop("hparams", None) or ModelBase.load_hparams(dir_model, self.is_mistral_format))
-
-        self.block_configs: list[dict] = hparams["block_configs"]
-        self.n_layer_trunk = len(self.block_configs)
-
-        # block_configs carries the per-block MoE shape, and is the authority on the
-        # block pattern too: the layers_block_type the HF config wrapper computes is
-        # not sized to it.
-        hparams["num_hidden_layers"] = self.n_layer_trunk
-        hparams["layers_block_type"] = [bc["block_type"] for bc in self.block_configs]
-
-        self.model_arch = gguf.MODEL_ARCH.NEMOTRON_H_MOE
-
-        # Bypass NemotronHModel.__init__: it assumes a flat num_experts_per_tok /
-        # moe_intermediate_size and a layers_block_type sized to block_count, neither
-        # of which hold for Puzzle's per-block config.
-        GraniteHybridModel.__init__(self, dir_model, *args, hparams=hparams, **kwargs)
-
-        self.head_dim = self.find_hparam(["head_dim", "attention_head_dim"])
-        self.d_inner = self.find_hparam(["num_heads"]) * self.d_model
-
-        # NemotronHModel.__init__ folds an MTP block into block_count when the
-        # config carries num_nextn_predict_layers; Puzzle's config does, but its
-        # head has a different layout and no inference path, so stay opted out.
-        self._mtp_bid = None
-
-    def set_gguf_parameters(self):
-        GraniteHybridModel.set_gguf_parameters(self)
-
-        head_dim = self.head_dim
-        if head_dim is None:
-            raise ValueError("Could not find the attention head dim in config")
-        self.gguf_writer.add_key_length(head_dim)
-        self.gguf_writer.add_value_length(head_dim)
-
-        ffn_lengths = [bc.get("moe_intermediate_size") or 0 for bc in self.block_configs]
-        experts_used = [bc.get("num_experts_per_tok") or 0 for bc in self.block_configs]
-
-        self.gguf_writer.add_feed_forward_length(ffn_lengths)
-        self.gguf_writer.add_expert_feed_forward_length(ffn_lengths)
-        self.gguf_writer.add_expert_used_count(experts_used)
-
-        self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])
-        self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])
-        self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])
-        self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
-        self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
-        self.gguf_writer.add_expert_group_count(self.hparams["n_group"])
-        self.gguf_writer.add_moe_latent_size(self.hparams["moe_latent_size"])
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # The official BF16 checkpoint (NVIDIA-Nemotron-Labs-3-Puzzle-75B-A9B-BF16)
-        # names the trunk "model.*" (model.layers.*, model.embeddings, model.norm_f)
-        # where the original release used the NemotronH-style "backbone.*", and spells
-        # the router bias "e_score_correction_bias" instead of "e_score_correction.bias";
-        # normalize so both convert identically.
-        if name.startswith("model."):
-            name = "backbone." + name[len("model."):]
-        if name.endswith("mixer.gate.e_score_correction_bias"):
-            name = name[: -len("e_score_correction_bias")] + "e_score_correction.bias"
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        # Drop the MTP head unconditionally; see the class docstring.
-        if item[0].startswith("mtp."):
-            return None
-        return super().filter_tensors(item)

conversion/qwen.py

@@ -379,13 +379,6 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
         self.gguf_writer.add_ssm_group_count(self.hparams["linear_num_key_heads"])
         self.gguf_writer.add_ssm_time_step_rank(self.hparams["linear_num_value_heads"])
         self.gguf_writer.add_ssm_inner_size(self.hparams["linear_value_head_dim"] * self.hparams["linear_num_value_heads"])
-        if (layer_types := self.hparams.get("layer_types")) is not None:
-            n_layer = self.hparams["num_hidden_layers"]
-            if len(layer_types) != n_layer:
-                raise ValueError(f"layer_types has {len(layer_types)} entries, expected num_hidden_layers ({n_layer})")
-            recurrent = [t == "linear_attention" for t in layer_types]
-            recurrent += [False] * (self.block_count - n_layer)
-            self.gguf_writer.add_recurrent_layers(recurrent)
         self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4))
         if (rope_dim := self.hparams.get("head_dim")) is None:
             rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
@@ -646,7 +639,7 @@ class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
     model_arch = gguf.MODEL_ARCH.QWEN35MOE
 
 
-@ModelBase.register("DFlashDraftModel", "DFlash2DraftModel")
+@ModelBase.register("DFlashDraftModel")
 @ModelBase.example("z-lab/Qwen3.5-9B-DFlash")
 class DFlashModel(Qwen3Model):
     model_arch = gguf.MODEL_ARCH.DFLASH
@@ -685,98 +678,34 @@ class DFlashModel(Qwen3Model):
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
 
-        dflash_config = self.hparams.get("dflash_config", {})
-        block_size = dflash_config.get("block_size", self.hparams.get("block_size", 16))
+        block_size = self.hparams.get("block_size", 16)
         self.gguf_writer.add_block_size(block_size)
-
-        if "conv_kernel_size" in dflash_config:
-            self.gguf_writer.add_conv_kernel_size(int(dflash_config["conv_kernel_size"]))
-            self.gguf_writer.add_conv_group_size(int(dflash_config["conv_group_size"]))
-            self.gguf_writer.add_selector_rank(int(dflash_config["selector_rank"]))
-            self.gguf_writer.add_selector_top_k(int(dflash_config["selector_top_k"]))
-
-        output_multiplier = dflash_config.get(
-            "output_multiplier", self.hparams.get("output_multiplier")
-        )
-        if output_multiplier is not None:
-            self.gguf_writer.add_logit_scale(float(output_multiplier))
-        softcap = dflash_config.get(
-            "final_logit_softcapping", self.hparams.get("final_logit_softcapping")
-        )
-        if softcap is not None and float(softcap) > 0:
-            self.gguf_writer.add_final_logit_softcapping(float(softcap))
-        embedding_scale = dflash_config.get(
-            "input_embedding_scale", self.hparams.get("input_embedding_scale")
-        )
-        if embedding_scale is not None:
-            self.gguf_writer.add_embedding_scale(float(embedding_scale))
+        dflash_config = self.hparams.get("dflash_config", {})
 
         target_layer_ids = dflash_config.get("target_layer_ids", [])
         if target_layer_ids:
             extract_layer_ids = [i + 1 for i in target_layer_ids]
             self.gguf_writer.add_target_layers(extract_layer_ids)
 
-        use_sliding_window = self.hparams.get("use_sliding_window", False) or dflash_config.get("use_swa", False)
-        sliding_window = dflash_config.get("swa_window_size") or self.hparams.get("sliding_window")
+        use_sliding_window = self.hparams.get("use_sliding_window", False)
+        sliding_window = self.hparams.get("sliding_window")
         layer_types = self.hparams.get("layer_types")
         if use_sliding_window and sliding_window and layer_types:
             is_swa = [lt == "sliding_attention" for lt in layer_types]
             self.gguf_writer.add_sliding_window(sliding_window)
             self.gguf_writer.add_sliding_window_pattern(is_swa)
 
-        causal = self.hparams.get("is_causal")
-        if causal is None:
-            causal = dflash_config.get("causal")
-        if causal is not None:
-            self.gguf_writer.add_causal_attention(bool(causal))
-
-        # M-RoPE target: the draft ropes on the temporal dim only, so write
-        # degenerate sections [n_rot/2, 0, 0, 0]
-        if self._target_uses_mrope():
-            head_dim = self.hparams.get("head_dim") or self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
-            self.gguf_writer.add_rope_dimension_sections([head_dim // 2, 0, 0, 0])
-
-    def _target_uses_mrope(self) -> bool:
-        if self.target_model_dir is None:
-            return False
-        with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:
-            cfg = json.load(f)
-        cfg = cfg.get("text_config", cfg)
-        rope = cfg.get("rope_parameters") or cfg.get("rope_scaling") or {}
-        return "mrope_section" in rope
-
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         name, gen = item
         if not name.startswith("model."):
             name = "model." + name
-        if "sink" in name and not name.endswith(".weight"):
-            name += ".weight"
         return super().filter_tensors((name, gen))
 
-    _ROPE_PERMUTE_SUFFIXES = (
-        "self_attn.q_proj.weight",
-        "self_attn.k_proj.weight",
-        "self_attn.q_norm.weight",
-        "self_attn.k_norm.weight",
-    )
-
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if name == "model.embed_tokens.weight" and not self.hparams.get("has_embed_tokens", True):
             return
 
-        # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd
-        if not self.hparams.get("rope_is_neox_style", True) and name.endswith(self._ROPE_PERMUTE_SUFFIXES):
-            head_dim = self.hparams["head_dim"]
-            shape = data_torch.shape
-            data_torch = data_torch.reshape(-1, head_dim // 2, 2, *shape[1:]).transpose(1, 2).reshape(shape)
-
-        if name in (
-            "model.candidate_selector.predecessor_codebook",
-            "model.candidate_selector.successor_codebook",
-        ):
-            name += ".weight"
-
         yield from super().modify_tensors(data_torch, name, bid)
 
 
@@ -830,10 +759,6 @@ class DSparkModel(DFlashModel):
         super().set_gguf_parameters()
         self.gguf_writer.add_sample_from_anchor(self._sample_from_anchor)
 
-        # confidence head is optional: vanilla-markov exports ship without it
-        has_conf = any("confidence_head.proj" in name for name in self.model_tensors)
-        self.gguf_writer.add_has_confidence_head(has_conf)
-
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         if item[0] == "t2d":  # not used at runtime
@@ -852,7 +777,7 @@ class DSparkModel(DFlashModel):
             self._d2t = data_torch
             return
 
-        if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith("lm_head.weight"):
+        if self._n_vocab_draft == self.hparams["vocab_size"] and name.endswith(("embed_tokens.weight", "lm_head.weight")):
             return
 
         # interleaved-rope checkpoints (rope_is_neox_style = false) -> NeoX layout: per head, even dims first then odd

conversion/qwen3tts.py

@@ -276,10 +276,6 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
         # ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
         if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
             return gguf.GGMLQuantizationType.F32
-        # the code predictor FFN intermediate peaks around 1.5e5, above the F16 range, and mul_mat
-        # casts its input to the weight type
-        if new_name.startswith("a.gen.code.blk.") and new_name.endswith(".ffn_down.weight"):
-            return gguf.GGMLQuantizationType.F32
         return super().tensor_force_quant(name, new_name, bid, n_dims)
 
     @classmethod

conversion/qwen4exp.py

@@ -1,195 +0,0 @@
-from __future__ import annotations
-
-from typing import Iterable, cast
-
-import torch
-from torch import Tensor
-
-import gguf
-import numpy as np
-
-from .base import ModelBase
-from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin
-from .qwen3vl import Qwen3VLVisionModel
-
-
-@ModelBase.register("Qwen4ExpForConditionalGeneration", "Qwen4ExpForCausalLM")
-@ModelBase.example("Qwen/Qwen3.8-Flash-Next")
-class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
-    """Qwen3.8-Flash-Next.
-
-    Shares the Qwen3.5 gated delta net and interleaved mrope, and adds three things:
-    hyper-connections in place of every layer norm, QSA sparse attention on the full
-    attention layers, and PLE n-gram hash embeddings on a single layer.
-    """
-
-    model_arch = gguf.MODEL_ARCH.QWEN4EXP
-
-    # the MTP block is a separate draft head; vLLM drops it too
-    supports_mtp_export = False
-    no_mtp = True
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # only the shard names, so the table itself is never held
-        self._ple_shards: dict[int, str] = {}
-        self._ple_row_dim: int | None = None
-
-    def _read_hash_constants(self, suffix: str) -> list[int]:
-        """Read an int64 PLE constant straight from the checkpoint.
-
-        prepare_tensors() casts every non-float dtype to float32 before
-        modify_tensors() sees it (base.py), which would silently round these
-        45-bit multipliers. Reading the lazy tensor here bypasses that.
-        """
-        for name, gen in self.model_tensors.items():
-            if name.endswith(suffix):
-                t = gen()
-                if t.dtype != torch.int64:
-                    t = t.to(torch.int64)
-                return [int(x) for x in t.tolist()]
-        raise ValueError(f"PLE constant {suffix!r} missing from the checkpoint")
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        hp = self.hparams
-
-        self.gguf_writer.add_hyper_connection_count(hp["hc_count"])
-        self.gguf_writer.add_hyper_connection_low_rank(hp["hc_lowrank"])
-
-        n_layer = hp["num_hidden_layers"]
-        self.gguf_writer.add_indexer_head_count(hp["indexer_n_heads"])
-        self.gguf_writer.add_indexer_key_length(hp["indexer_head_dim"])
-        self.gguf_writer.add_indexer_top_k(hp["indexer_budget"])
-        ratio = hp["indexer_compress_ratio"]
-        layer_types = hp["layer_types"]
-        self.gguf_writer.add_attention_compress_ratios(
-            [ratio if layer_types[i] == "full_attention" else 0 for i in range(n_layer)]
-        )
-
-        # ple_layer_ids is 1-based in the HF config; empty means no n-gram table,
-        # so emit no PLE keys rather than optional ones
-        ple_layers = [i - 1 for i in hp["ple_layer_ids"]]
-        if not ple_layers:
-            return
-        self.gguf_writer.add_ple_layers(ple_layers)
-        self.gguf_writer.add_ple_ngram_size(hp["ngram_size"])
-        self.gguf_writer.add_ple_heads_per_ngram(hp["heads_per_ngram"])
-        self.gguf_writer.add_ple_conv_kernel(hp["ple_conv_kernel_size"])
-        self.gguf_writer.add_ple_eos_token_id(self._eos_token_id())
-        # an image is decoded as an embeddings-only batch, so the graph has no placeholder
-        # ids to hash; carry the id and let it stand in for those positions
-        _img = self._image_token_id()
-        if _img is not None:
-            self.gguf_writer.add_ple_image_token_id(int(_img))
-        if self._ple_row_dim is not None:
-            self.gguf_writer.add_embedding_length_per_layer_input(self._ple_row_dim)
-
-        self.gguf_writer.add_ple_layer_multipliers(
-            self._read_hash_constants("ple_embedding.layer_multipliers"))
-        self.gguf_writer.add_ple_head_offsets(
-            self._read_hash_constants("ple_embedding.ngram_heads_offsets"))
-        self.gguf_writer.add_ple_head_vocab_sizes(
-            self._read_hash_constants("ple_embedding.ngram_heads_vocab_sizes"))
-
-    def _image_token_id(self) -> int | None:
-        img = self.hparams.get("image_token_id")
-        return None if img is None else int(img)
-
-    def _eos_token_id(self) -> int:
-        eos = self.hparams.get("eos_token_id")
-        if isinstance(eos, list):
-            # the PLE hash resets n-grams on the primary EOS
-            return int(eos[-1])
-        if eos is None:
-            raise ValueError("eos_token_id is required: the PLE hash resets its n-grams on it")
-        return int(eos)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # int64 hash constants must stay exact; 1-D tensors force F32, so use KV
-        if name.endswith("ple_embedding.layer_multipliers"):
-            self._ple_multipliers = [int(x) for x in data_torch.tolist()]
-            return []
-        if name.endswith("ple_embedding.ngram_heads_offsets"):
-            self._ple_head_offsets = [int(x) for x in data_torch.tolist()]
-            return []
-        if name.endswith("ple_embedding.ngram_heads_vocab_sizes"):
-            self._ple_head_vocab_sizes = [int(x) for x in data_torch.tolist()]
-            return []
-
-        if ".ngram_embedding.shard_" in name:
-            return self._place_ple_shard(data_torch, name)
-
-        # one projection feeds indexer q and k; split it, as minimax-m3 does
-        if ".indexer.index_qk_proj.weight" in name:
-            n_q = self.hparams["indexer_n_heads"] * self.hparams["indexer_head_dim"]
-            q = data_torch[:n_q]
-            k = data_torch[n_q:]
-            return [
-                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), q),
-                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), k),
-            ]
-
-        # Gemma zero-centred gammas the inherited norm.weight rule misses
-        if name.endswith((".ple.norm_key.weight", ".ple.norm_query.weight", ".ple.norm_conv.weight",
-                          ".indexer.q_layernorm.weight", ".indexer.k_layernorm.weight")):
-            return [(self.map_tensor_name(name), data_torch + 1)]
-
-        if name.endswith(".ple.conv1d.weight"):
-            return [(self.map_tensor_name(name), data_torch.squeeze())]
-
-        return super().modify_tensors(data_torch, name, bid)
-
-    # the shards concatenate into a tensor of well over 100 GB
-    # use LazyChunkedTensor here, a single shard resident at a time
-    def _place_ple_shard(self, data_torch: Tensor, name: str) -> Iterable[tuple[str, Tensor]]:
-
-        idx = int(name.rpartition(".shard_")[2].partition(".")[0])
-        n_parts = self.hparams["split_ngram_parts"]
-
-        self._ple_shards[idx] = name
-        self._ple_row_dim = int(data_torch.shape[-1])
-
-        if len(self._ple_shards) < n_parts:
-            return []
-
-        # the checkpoint may yield the shards in any order, the row order is by index
-        shards = [self._ple_shards[i] for i in sorted(self._ple_shards)]
-        rows = 0
-        for shard in shards:
-            shape = self.model_tensors[shard]().shape
-            if int(shape[-1]) != self._ple_row_dim:
-                raise ValueError(
-                    f"PLE shard {shard} has row dim {int(shape[-1])}, expected {self._ple_row_dim}")
-            rows += int(shape[0])
-
-        table = gguf.LazyChunkedTensor(
-            [self._load_ple_shard(shard) for shard in shards],
-            shape=(rows, self._ple_row_dim),
-            dtype=np.float32,
-        )
-        gguf_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
-        return [(gguf_name + ".weight", cast(Tensor, table))]
-
-    def _load_ple_shard(self, name: str):
-        def load() -> np.ndarray:
-            from .base import LazyTorchTensor
-
-            # a fresh lazy tensor every call, or to_eager() memoizes every shard
-            eager = LazyTorchTensor.to_eager(self.model_tensors[name]())
-            return eager.to(torch.float32).contiguous().numpy()
-        return load
-
-    def prepare_tensors(self):
-        super().prepare_tensors()
-        n_parts = self.hparams.get("split_ngram_parts", 0)
-        if self._ple_shards and len(self._ple_shards) != n_parts:
-            raise ValueError(
-                f"got {len(self._ple_shards)} PLE embedding shards, expected {n_parts}"
-            )
-
-
-@ModelBase.register("Qwen4ExpForConditionalGeneration")
-@ModelBase.example("Qwen/Qwen3.8-Flash-Next")
-class Qwen4ExpVisionModel(Qwen3VLVisionModel):
-    """The vision tower is an unmodified Qwen3-VL ViT."""

conversion/spark2_5.py

@@ -1,65 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Iterable
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, TextModel, gguf
-
-
-@ModelBase.register("Spark2_5ForCausalLM")
-@ModelBase.example("XHToken/Spark-X2.5-1.7B")
-class Spark2_5Model(TextModel):
-    model_arch = gguf.MODEL_ARCH.SPARK2_5
-
-    def set_gguf_parameters(self) -> None:
-        super().set_gguf_parameters()
-
-        hparams = self.hparams
-        layer_types = hparams["layer_types"]
-        if len(layer_types) != self.block_count:
-            raise ValueError(
-                f"Spark2_5 layer_types length {len(layer_types)} != num_hidden_layers {self.block_count}"
-            )
-        if any(layer_type not in ("sliding_attention", "full_attention") for layer_type in layer_types):
-            raise ValueError(f"Spark2_5 has unsupported layer_types: {layer_types}")
-        if hparams.get("gate_attn_act_mode") != "sigmoid" or hparams.get("headwise_attn_output_gate") is not True:
-            raise ValueError("Spark2_5 conversion requires head-wise sigmoid attention gates")
-        if hparams.get("hidden_act") != "gelu":
-            raise ValueError(f"Spark2_5 conversion requires GELU, got {hparams.get('hidden_act')!r}")
-
-        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
-        self.gguf_writer.add_sliding_window(hparams["sliding_window"])
-        self.gguf_writer.add_sliding_window_pattern(
-            [layer_type == "sliding_attention" for layer_type in layer_types]
-        )
-
-        head_dim = hparams["head_dim"]
-        full_rope = self.rope_parameters["full_attention"]
-        swa_rope = self.rope_parameters["sliding_attention"]
-        self.gguf_writer.add_rope_dimension_count(
-            int(head_dim * float(full_rope["partial_rotary_factor"]))
-        )
-        self.gguf_writer.add_rope_dimension_count_swa(
-            int(head_dim * float(swa_rope["partial_rotary_factor"]))
-        )
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".self_attn.q_k_v_proj.weight"):
-            if bid is None:
-                raise ValueError(f"Spark2_5 fused QKV tensor has no block id: {name}")
-            yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid), data_torch
-            return
-
-        if name.endswith(".self_attn.g_proj.weight"):
-            if bid is None:
-                raise ValueError(f"Spark2_5 attention gate tensor has no block id: {name}")
-            expected = self.hparams["num_attention_heads"]
-            if data_torch.shape[0] != expected:
-                raise ValueError(
-                    f"Spark2_5 layer {bid} attention gate width {data_torch.shape[0]} != head count {expected}"
-                )
-
-        yield from super().modify_tensors(data_torch, name, bid)