ci : run the server-speculative steps with bash

.devops/intel.Dockerfile

@@ -64,7 +64,7 @@ RUN mkdir -p /app/full \
     && cp requirements.txt /app/full \
     && cp .devops/tools.sh /app/full/tools.sh
 
-FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base
+FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base
 
 ARG BUILD_DATE=N/A
 ARG APP_VERSION=N/A
@@ -79,8 +79,6 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
       org.opencontainers.image.url=$IMAGE_URL \
       org.opencontainers.image.source=$IMAGE_SOURCE
 
-COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl
-
 #Following versions are for multiple GPUs, since 26.x has known issue:
 #   https://github.com/ggml-org/llama.cpp/issues/21747,
 #   https://github.com/intel/compute-runtime/issues/921.

.github/actions/windows-setup-cuda/action.yml

@@ -100,56 +100,56 @@ runs:
           echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
           echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
 
-    - name: Install Cuda Toolkit 13.5 for x64
-      if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'x64' }}
+    - name: Install Cuda Toolkit 13.4 for x64
+      if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }}
       shell: pwsh
       run: |
-          mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
+          mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
           choco install unzip -y
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.10.0.20-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.5.25-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
-          unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvrtc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-x86_64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvtx-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_profiler_api-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\visual_studio_integration-windows-x86_64-13.5.25-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
-          echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.4.49-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.4.59-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.7.0.27-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.4.49-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.4.49-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.4.49-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
+          unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvrtc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-x86_64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvtx-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_profiler_api-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\visual_studio_integration-windows-x86_64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
 
-    - name: Install Cuda Toolkit 13.5 for ARM64
-      if: ${{ inputs.cuda_version == '13.5' && inputs.cuda_arch == 'arm64' }}
+    - name: Install Cuda Toolkit 13.4 for ARM64
+      if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'arm64' }}
       shell: pwsh
       run: |
-          mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
+          mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
           choco install unzip -y
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
-          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
-          unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libnvvm-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\libcublas-windows-arm64-13.10.0.20-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
-          echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
-          echo "CUDA_PATH_V13_5=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.4.2.1-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.4.59-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.4.59-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.4.59-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.4.49-archive.zip"
+          curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.7.0.27-archive.zip"
+          unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4"
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cccl-windows-x86_64-13.3.4.2.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_crt-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_nvcc-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libnvvm-windows-x86_64-13.4.59-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\cuda_cudart-windows-arm64-13.4.49-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\libcublas-windows-arm64-13.7.0.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" /E /I /H /Y
+          echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+          echo "CUDA_PATH_V13_4=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8

.github/workflows/behavior-check.yml

@@ -1,172 +0,0 @@
-name: Behavior Check
-
-on:
-  workflow_dispatch: # allows manual triggering
-  push:
-    branches:
-      - master
-    paths: [
-      '.github/workflows/behavior-check.yml',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp',
-      '**/*.py',
-      '**/requirements*.txt',
-    ]
-
-  pull_request:
-    types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/behavior-check.yml',
-      '**/CMakeLists.txt',
-      '**/.cmake',
-      '**/*.h',
-      '**/*.hpp',
-      '**/*.c',
-      '**/*.cpp',
-      '**/*.py',
-      '**/requirements*.txt',
-    ]
-
-cache-mode: none
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  GGML_SCHED_DEBUG_REALLOC: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  ubuntu:
-    runs-on: ubuntu-26.04-arm
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: Build Dependencies
-        id: build_depends
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential libssl-dev git-lfs libjpeg-dev python3 python3-venv python3-pip
-
-      - name: Python Dependencies
-        id: python_depends
-        run: |
-          python3 -m venv .venv
-          source .venv/bin/activate
-          pip install -r requirements/requirements-convert_hf_to_gguf.txt
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: behavior-check-ubuntu-26.04-arm
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DGGML_NATIVE=OFF \
-            -DGGML_BACKEND_DL=OFF \
-            -DLLAMA_FATAL_WARNINGS=ON
-          time cmake --build build --config Release -j $(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: behavior-check-ubuntu-26.04-arm
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
-      - name: Test
-        id: cmake_test
-        run: |
-          source .venv/bin/activate
-          cd build
-          ctest -L behavior -LE 'model|backend' --verbose --timeout 900
-
-  windows:
-    runs-on: windows-2025
-
-    cache-mode: write
-    permissions:
-      actions: write
-      contents: read
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          key: behavior-check-windows-2025
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-
-      - name: Python Dependencies
-        id: python_depends
-        shell: bash
-        run: |
-          python3 -m venv .venv
-          source .venv/Scripts/activate
-          pip install -r requirements/requirements-convert_hf_to_gguf.txt
-
-      - name: Build
-        id: cmake_build
-        shell: cmd
-        run: |
-          call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
-          cmake -B build -G "Ninja Multi-Config" ^
-            -DCMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake ^
-            -DCMAKE_BUILD_TYPE=Release ^
-            -DGGML_NATIVE=OFF ^
-            -DGGML_BACKEND_DL=OFF
-          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
-          cmake --build build --config Release -j %NINJA_JOBS%
-
-      - name: Test
-        id: cmake_test
-        shell: bash
-        run: |
-          source .venv/Scripts/activate
-          cd build
-          ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: behavior-check-windows-2025
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/build-apple-daily.yml

@@ -1,160 +0,0 @@
-name: CI (apple - daily)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  schedule:
-    - cron: '0 0 * * *'
-
-cache-mode: none
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  macos-latest-ios-xcode:
-    runs-on: macos-latest
-
-    permissions:
-      actions: write
-      contents: read
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-
-      - name: Setup Xcode
-        uses: ggml-org/setup-xcode@v1
-        with:
-          xcode-version: latest-stable
-
-      - name: Build
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=iOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-
-      - name: xcodebuild for swift package
-        id: xcodebuild
-        run: |
-          ./build-xcframework.sh
-
-      - name: Upload xcframework artifact
-        uses: actions/upload-artifact@v6
-        with:
-          name: llama-xcframework
-          path: build-apple/llama.xcframework/
-          retention-days: 1
-
-      - name: Build Xcode project
-        run: |
-          xcodebuild -downloadPlatform iOS
-          xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
-
-  macos-latest-tvos:
-    runs-on: macos-latest
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Build
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_BUILD_COMMON=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=tvOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-
-  macos-latest-visionos:
-    runs-on: macos-latest
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Build
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_BUILD_COMMON=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=visionOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-
-  macos-latest-swift:
-    runs-on: macos-latest
-    needs: macos-latest-ios-xcode
-
-    permissions:
-      actions: read
-      contents: read
-
-    strategy:
-      matrix:
-        destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Download xcframework artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: llama-xcframework
-          path: build-apple/llama.xcframework/
-
-      - name: Build llama.cpp with CMake
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

.github/workflows/build-apple.yml

@@ -105,5 +105,198 @@ jobs:
           cd build
           # Metal Paravirtual devices are difficult to support -> disable
           # ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
-          ctest -L main -E "test-llama-archs|test-llama-context|test-speculative-rejection-sampling" --verbose --timeout 900
+          ctest -L main -E "test-llama-archs|test-llama-context" --verbose --timeout 900
 
+  macos-latest-x64:
+    runs-on: macos-15-intel
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.24
+        with:
+          restore: false
+          save: false
+
+      - name: ccache-buckets-restore
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          hf_bucket: ggml-org/cache
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          # Metal is disabled due to intermittent failures with Github runners not having a GPU:
+          # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
+          cmake -B build \
+            -DCMAKE_BUILD_RPATH="@loader_path" \
+            -DLLAMA_FATAL_WARNINGS=ON \
+            -DLLAMA_BUILD_BORINGSSL=ON \
+            -DGGML_METAL=OFF \
+            -DGGML_RPC=ON \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
+          time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
+
+      - name: ccache-buckets-save
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+        uses: ./.github/actions/ccache-buckets
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
+        with:
+          key: apple-x64
+          folder: llama.cpp
+          evict-old-files: 1d
+          hf_bucket: ggml-org/cache
+          save: true
+
+      - name: Test
+        id: cmake_test
+        run: |
+          cd build
+          ctest -L main --verbose --timeout 900
+
+  macos-latest-ios-xcode:
+    runs-on: macos-latest
+
+    permissions:
+      actions: write
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Xcode
+        uses: ggml-org/setup-xcode@v1
+        with:
+          xcode-version: latest-stable
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_OPENSSL=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_SYSTEM_NAME=iOS \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+      - name: xcodebuild for swift package
+        id: xcodebuild
+        run: |
+          ./build-xcframework.sh
+
+      - name: Upload xcframework artifact
+        uses: actions/upload-artifact@v6
+        with:
+          name: llama-xcframework
+          path: build-apple/llama.xcframework/
+          retention-days: 1
+
+      - name: Build Xcode project
+        run: |
+          xcodebuild -downloadPlatform iOS
+          xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
+
+  macos-latest-tvos:
+    runs-on: macos-latest
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_BUILD_COMMON=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_SYSTEM_NAME=tvOS \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+  macos-latest-visionos:
+    runs-on: macos-latest
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_BUILD_COMMON=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_SYSTEM_NAME=visionOS \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
+            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+  macos-latest-swift:
+    runs-on: macos-latest
+    needs: macos-latest-ios-xcode
+
+    permissions:
+      actions: read
+      contents: read
+
+    strategy:
+      matrix:
+        destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Download xcframework artifact
+        uses: actions/download-artifact@v7
+        with:
+          name: llama-xcframework
+          path: build-apple/llama.xcframework/
+
+      - name: Build llama.cpp with CMake
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_OPENSSL=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

.github/workflows/build-cmake-pkg.yml

@@ -0,0 +1,57 @@
+name: Build relocatable cmake package
+on:
+  workflow_dispatch:
+  workflow_call:
+
+cache-mode: none
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    runs-on: [self-hosted, Linux, CPU]
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Build
+        run: |
+          PREFIX="$(pwd)"/inst
+          cmake -S . -B build \
+                -DCMAKE_PREFIX_PATH="$PREFIX" \
+                -DLLAMA_OPENSSL=OFF \
+                -DLLAMA_BUILD_TESTS=OFF \
+                -DLLAMA_BUILD_TOOLS=OFF \
+                -DLLAMA_BUILD_EXAMPLES=OFF \
+                -DLLAMA_BUILD_APP=OFF \
+                -DLLAMA_BUILD_IS_DEV=OFF \
+                -DCMAKE_BUILD_TYPE=Release
+          cmake --build build --config Release -j $(nproc)
+          cmake --install build --prefix "$PREFIX" --config Release
+
+          export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
+          build_commit=$(git rev-parse --short HEAD | xargs)
+          build_number=$(git rev-list --count HEAD | xargs)
+
+          major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
+          minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
+          patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
+          build_version="$major.$minor.$patch"
+
+          checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
+                  "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
+                  "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
+
+          for check in "${checks[@]}"; do
+              if ! grep -qE "$check" "$LLAMA_CONFIG"; then
+                  echo "Checking llama-config.cmake version... \"$check\" failed!"
+                  exit 1
+              fi
+          done
+
+          echo "Checking llama-config.cmake version... success."
+
+          cd examples/simple-cmake-pkg
+          cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
+          cmake --build build -j $(nproc)

.github/workflows/build-cpu.yml

@@ -7,6 +7,7 @@ on:
       - master
     paths: [
       '.github/workflows/build-cpu.yml',
+      '.github/workflows/build-cmake-pkg.yml',
       '**/CMakeLists.txt',
       '**/.cmake',
       '**/*.h',
@@ -19,6 +20,7 @@ on:
     types: [opened, synchronize, reopened]
     paths: [
       '.github/workflows/build-cpu.yml',
+      '.github/workflows/build-cmake-pkg.yml',
       'ggml/src/ggml-rpc/**',
       '**/CMakeLists.txt',
       '**/.cmake',
@@ -45,6 +47,9 @@ env:
   LLAMA_ARG_LOG_TIMESTAMPS: 1
 
 jobs:
+  build-cmake-pkg:
+    uses: ./.github/workflows/build-cmake-pkg.yml
+
   ubuntu:
     strategy:
       matrix:
@@ -124,7 +129,7 @@ jobs:
         id: cmake_test
         run: |
           cd build
-          ctest -L main --verbose --timeout 900
+          ctest -L 'main|python' --verbose --timeout 900
 
       - name: Test llama2c conversion
         id: llama2c_test
@@ -237,79 +242,3 @@ jobs:
           older: 5m
           min: 1
           dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
-
-  build-cmake-pkg:
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Clone
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: build-cmake-pkg
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
-      - name: Build
-        run: |
-          PREFIX="$(pwd)"/inst
-          cmake -S . -B build \
-                -DCMAKE_PREFIX_PATH="$PREFIX" \
-                -DLLAMA_OPENSSL=OFF \
-                -DLLAMA_BUILD_TESTS=OFF \
-                -DLLAMA_BUILD_TOOLS=OFF \
-                -DLLAMA_BUILD_EXAMPLES=OFF \
-                -DLLAMA_BUILD_APP=OFF \
-                -DLLAMA_BUILD_IS_DEV=OFF \
-                -DCMAKE_BUILD_TYPE=Release
-          cmake --build build --config Release -j $(nproc)
-          cmake --install build --prefix "$PREFIX" --config Release
-
-          export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
-          build_commit=$(git rev-parse --short HEAD | xargs)
-          build_number=$(git rev-list --count HEAD | xargs)
-
-          major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
-          minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
-          patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
-          build_version="$major.$minor.$patch"
-
-          checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
-                  "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
-                  "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
-
-          for check in "${checks[@]}"; do
-              if ! grep -qE "$check" "$LLAMA_CONFIG"; then
-                  echo "Checking llama-config.cmake version... \"$check\" failed!"
-                  exit 1
-              fi
-          done
-
-          echo "Checking llama-config.cmake version... success."
-
-          cd examples/simple-cmake-pkg
-          cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
-          cmake --build build -j $(nproc)
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: build-cmake-pkg
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true

.github/workflows/build-cuda-windows.yml

@@ -40,12 +40,12 @@ jobs:
           - cuda: '12.4'
             arch: x64
             defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
-          - cuda: '13.5'
+          - cuda: '13.4'
             arch: x64
-            defines: ''
-          - cuda: '13.5'
+            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
+          - cuda: '13.4'
             arch: arm64
-            defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
+            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
 
     steps:
       - name: Clone

.github/workflows/build-ibm.yml

@@ -43,9 +43,9 @@ env:
   LLAMA_ARG_LOG_TIMESTAMPS: 1
 
 jobs:
-  ubuntu-26-s390x:
-    name: ubuntu-26-s390x (VXE ${{ matrix.vxe }})
-    runs-on: ubuntu-26.04-s390x
+  ubuntu-24-s390x:
+    name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
+    runs-on: ubuntu-24.04-s390x
 
     strategy:
       fail-fast: false
@@ -66,6 +66,12 @@ jobs:
             libjpeg-dev build-essential libssl-dev \
             git-lfs
 
+      - name: Toolchain workaround (GCC 14)
+        run: |
+          sudo apt-get install -y gcc-14 g++-14
+          echo "CC=gcc-14" >> "$GITHUB_ENV"
+          echo "CXX=g++-14" >> "$GITHUB_ENV"
+
       - name: Python Dependencies
         id: python_depends
         run: |
@@ -110,19 +116,30 @@ jobs:
 
   ubuntu-26-zdnn-s390x:
     name: ubuntu-26-zdnn-s390x
-    runs-on: ubuntu-26.04-s390x
+    runs-on: ubuntu-24.04-s390x
+    container: ubuntu:26.04  # required to get GCC 15.1 and binutils 2.44
+    defaults:
+      run:
+        shell: bash
 
     steps:
+      - name: Build Dependencies
+        id: build_depends
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends \
+            build-essential cmake git ca-certificates \
+            libssl-dev libzdnn-dev
+
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
 
-      - name: Build Dependencies
-        id: build_depends
+      - name: Toolchain workaround (GCC 15)
         run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential libssl-dev libzdnn-dev
+          apt-get install -y gcc-15 g++-15
+          echo "CC=gcc-15" >> "$GITHUB_ENV"
+          echo "CXX=g++-15" >> "$GITHUB_ENV"
 
       - name: Build with zDNN Backend
         id: cmake_build

.github/workflows/build-sanitize.yml

@@ -43,12 +43,13 @@ jobs:
     strategy:
       matrix:
         include:
+        # thread and address doesn't run properly on some self hosted machines, so run it on Github instead
         - sanitizer: ADDRESS
-          machine: ubuntu-26.04
+          machine: ubuntu-24.04
         - sanitizer: THREAD
-          machine: ubuntu-26.04
+          machine: ubuntu-24.04
         - sanitizer: UNDEFINED
-          machine: ubuntu-26.04
+          machine: [self-hosted, X64, Linux]
 
     runs-on: ${{ matrix.machine }}
 
@@ -57,24 +58,31 @@ jobs:
         id: checkout
         uses: actions/checkout@v6
 
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
+      # - name: ccache
+      #   uses: ggml-org/ccache-action@v1.2.24
+      #   if: ${{ matrix.sanitizer != 'UNDEFINED' }}
+      #   with:
+      #     key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
+      #     variant: ccache
+      #     evict-old-files: 1d
+      #     save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
-        with:
-          key: ctest-sanitizer-${{ matrix.sanitizer }}
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
+      # with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings
+      - name: Build (undefined)
+        id: cmake_build_undefined
+        if: ${{ matrix.sanitizer == 'UNDEFINED' }}
+        run: |
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DLLAMA_FATAL_WARNINGS=ON \
+            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+            -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
+
+          cmake --build build --config Debug -j $(nproc)
 
       - name: Build
         id: cmake_build
-        if: ${{ matrix.sanitizer == 'ADDRESS' || matrix.sanitizer == 'UNDEFINED' }}
+        if: ${{ matrix.sanitizer == 'ADDRESS' }}
         run: |
           cmake -B build \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
@@ -95,18 +103,6 @@ jobs:
 
           cmake --build build --config RelWithDebInfo -j $(nproc)
 
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: ctest-sanitizer-${{ matrix.sanitizer }}
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
-
       - name: Test
         id: cmake_test
         run: |

.github/workflows/ci-self-hosted-cpu.yml

@@ -46,54 +46,17 @@ env:
 
 jobs:
   cpu-x64-high-perf:
-    runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
-
-    # note: this runner has 8 CPUs, but std::thread::hardware_concurrency reports the cores of the
-    #   host (https://github.com/ggml-org/llama.cpp/pull/31439#issuecomment-6683386059)
-    env:
-      TEST_ARG_HARDWARE_CONCURRENCY: 8
-      TEST_ARG_PARALLEL_WORKERS: 4
+    runs-on: [self-hosted, Linux, X64]
 
     steps:
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
 
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt update
-          sudo apt install -y build-essential cmake libssl-dev time unzip wget python3 python3-venv python3-dev
-
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.24
-        with:
-          restore: false
-          save: false
-
-      - name: ccache-buckets-restore
-        uses: ./.github/actions/ccache-buckets
-        with:
-          key: self-hosted-cpu-x64-high-perf
-          folder: llama.cpp
-          hf_bucket: ggml-org/cache
-
       - name: Test
         id: ggml-ci
         run: |
-          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
-
-      - name: ccache-buckets-save
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
-        uses: ./.github/actions/ccache-buckets
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
-        with:
-          key: self-hosted-cpu-x64-high-perf
-          folder: llama.cpp
-          evict-old-files: 1d
-          hf_bucket: ggml-org/cache
-          save: true
+          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
 
   cpu-arm64-high-perf-graviton4:
     runs-on: ah-ubuntu_24_04-c8g_8x

.github/workflows/ci-self-hosted-metal.yml

@@ -60,4 +60,4 @@ jobs:
       - name: Test
         id: ggml-ci
         run: |
-          GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
+          GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/ci-self-hosted-vulkan.yml

@@ -79,7 +79,7 @@ jobs:
         id: ggml-ci
         run: |
           vulkaninfo --summary
-          GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
+          GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
 
       # - name: ccache-buckets-save
       #   if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
@@ -124,7 +124,7 @@ jobs:
         id: ggml-ci
         run: |
           vulkaninfo --summary
-          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
+          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
 
       # - name: ccache-buckets-save
       #   if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
@@ -150,7 +150,7 @@ jobs:
         id: ggml-ci
         run: |
           vulkaninfo --summary
-          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
+          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
 
   gpu-vulkan-intel-linux:
     runs-on: [self-hosted, Linux, Intel]

.github/workflows/ci-self-hosted-webgpu.yml

@@ -58,12 +58,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y software-properties-common
-          sudo add-apt-repository ppa:deadsnakes/ppa
-          sudo apt update
-          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3.12 python3.12-venv
-          sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
-          sudo update-alternatives --config python3
+          sudo apt install -y build-essential cmake libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libvulkan1 mesa-vulkan-drivers libglvnd0 libgl1 libglx0 libegl1 libgles2 libssl-dev time unzip wget python3 python3-venv python3-pip
 
       - name: ccache
         uses: ggml-org/ccache-action@v1.2.24
@@ -136,4 +131,4 @@ jobs:
         id: ggml-ci
         run: |
           GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
-            bash ./ci/run.sh ~/results/llama.cpp/$RUNNER_NAME ~/mnt/llama.cpp
+            bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp

.github/workflows/docker.yml

@@ -92,8 +92,8 @@ jobs:
             { "tag": "cpu", "dockerfile": ".devops/s390x.Dockerfile", "platforms": "linux/s390x", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04-s390x", "prebuilt_ui": true },
             { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
             { "tag": "cuda cuda12", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "12.8.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
-            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
-            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.5.0", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
+            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
+            { "tag": "cuda13", "dockerfile": ".devops/cuda.Dockerfile", "cuda_version": "13.4.1", "platforms": "linux/arm64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04-arm" },
             { "tag": "musa", "dockerfile": ".devops/musa.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
             { "tag": "intel", "dockerfile": ".devops/intel.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": true, "runs_on": "ubuntu-24.04" },
             { "tag": "vulkan", "dockerfile": ".devops/vulkan.Dockerfile", "platforms": "linux/amd64", "full": true, "light": true, "server": true, "free_disk_space": false, "runs_on": "ubuntu-24.04" },

.github/workflows/models-check.yml

@@ -11,8 +11,7 @@ on:
       'tests/fusion/**',
       'tests/test-fusion.cpp',
       'tests/test-llama-archs.cpp',
-      'src/llama-graph*',
-      'src/llama-kv-cache*',
+      'src/llama-graph.cpp',
       'src/llama-model*',
       'src/models/**'
     ]
@@ -25,8 +24,7 @@ on:
       'tests/fusion/**',
       'tests/test-fusion.cpp',
       'tests/test-llama-archs.cpp',
-      'src/llama-graph*',
-      'src/llama-kv-cache*',
+      'src/llama-graph.cpp',
       'src/llama-model*',
       'src/models/**'
     ]
@@ -147,11 +145,6 @@ jobs:
           rm -rf build-ci-models && mkdir -p build-ci-models
           ./build/bin/test-llama-archs -o build-ci-models
 
-      - name: Test KV shift
-        id: test_kv_shift
-        run: |
-          ./build/bin/test-llama-archs --kv-shift
-
       - name: Test fusion
         id: test_fusion
         run: |
@@ -300,11 +293,6 @@ jobs:
       #   run: |
       #     ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
 
-      - name: Test KV shift
-        id: test_kv_shift
-        run: |
-          ./build/bin/test-llama-archs --kv-shift
-
       - name: Test archs
         id: test_archs
         run: |

.github/workflows/pr-video-demo-label.yml

@@ -1,37 +0,0 @@
-name: "PR video demo label"
-
-on:
-  pull_request_target:
-    types: [opened, edited, reopened, synchronize]
-
-cache-mode: none
-permissions:
-  pull-requests: write
-  issues: write
-
-jobs:
-  video-demo-label:
-    runs-on: ubuntu-slim
-    steps:
-      - name: Add or remove the video demo label
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_URL: ${{ github.event.pull_request.html_url }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          # passed through env, so the body is only matched, never run, and is safe to read from a fork
-          BODY: ${{ github.event.pull_request.body }}
-          HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
-        run: |
-          # sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment
-          # markers, or a preview linking to a demo video uploaded for this PR
-          has_demo=false
-          if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
-            grep -qE "/resolve/videos/pr/$PR_NUMBER/[0-9a-f]{10}(-[0-9]+)?\.mp4" <<< "$BODY"; then
-            has_demo=true
-          fi
-
-          if [ "$has_demo" = true ] && [ "$HAS_LABEL" = false ]; then
-            gh pr edit "$PR_URL" --add-label 'video demo'
-          elif [ "$has_demo" = false ] && [ "$HAS_LABEL" = true ]; then
-            gh pr edit "$PR_URL" --remove-label 'video demo'
-          fi

.github/workflows/release-publish.yml

@@ -172,13 +172,11 @@ jobs:
             - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
             - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
             - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
-            - [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz)
-            - [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz)
             - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
             - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
             - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
-            - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-x64.tar.gz)
-            - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz) - [CUDA 13.5 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.5-arm64.tar.gz)
+            - [Ubuntu x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-x64.tar.gz)
+            - [Ubuntu arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz) - [CUDA 13.4 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-13.4-arm64.tar.gz)
             - [Ubuntu x64 (ROCm 10.0)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-rocm-10.0-x64.tar.gz)
             - [Ubuntu x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-2026.4.1-x64.tar.gz)
             - [Ubuntu x64 (SYCL FP32)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-sycl-fp32-x64.tar.gz)
@@ -194,8 +192,8 @@ jobs:
             - [Windows arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cpu-arm64.zip)
             - [Windows arm64 (OpenCL Adreno)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-opencl-adreno-arm64.zip)
             - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip)
-            - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-x64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-x64.zip)
-            - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.5-arm64.zip) - [CUDA 13.5 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.5-arm64.zip)
+            - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-x64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-x64.zip)
+            - [Windows arm64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.4-arm64.zip) - [CUDA 13.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.4-arm64.zip)
             - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip)
             - [Windows arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-arm64.zip)
             - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-2026.4.1-x64.zip)

.github/workflows/release.yml

@@ -180,8 +180,6 @@ jobs:
             os: ubuntu-24.04-arm
           - build: 's390x'
             os: ubuntu-24.04-s390x
-          - build: '26.04-s390x'
-            os: ubuntu-26.04-s390x
 
     runs-on: ${{ matrix.os }}
 
@@ -202,7 +200,7 @@ jobs:
         id: depends
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential libssl-dev
+          sudo apt-get install build-essential libssl-dev
 
       - name: Toolchain workaround (GCC 14)
         if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
@@ -212,7 +210,7 @@ jobs:
           echo "CXX=g++-14" >> "$GITHUB_ENV"
 
       - name: ccache
-        if: ${{ !contains(matrix.os, 's390x') }}
+        if: ${{ matrix.build != 's390x' }}
         uses: ggml-org/ccache-action@v1.2.24
         with:
           key: release-${{ matrix.os }}-cpu
@@ -244,7 +242,7 @@ jobs:
           archive: false
 
       - name: ccache-clear
-        if: ${{ !contains(matrix.os, 's390x') }}
+        if: ${{ matrix.build != 's390x' }}
         uses: ./.github/actions/ccache-clear
         with:
           key: release-${{ matrix.os }}-cpu
@@ -345,14 +343,14 @@ jobs:
             defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
           - build: 'x64'
             os: ubuntu-24.04
-            cuda: '13.5.0'
-            label: '13.5'
-            defines: ''
+            cuda: '13.4.1'
+            label: '13.4'
+            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
           - build: 'arm64'
             os: ubuntu-24.04-arm
-            cuda: '13.5.0'
-            label: '13.5'
-            defines: ''
+            cuda: '13.4.1'
+            label: '13.4'
+            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
 
     runs-on: ${{ matrix.os }}
     container: nvidia/cuda:${{ matrix.cuda }}-devel-ubuntu24.04
@@ -970,11 +968,6 @@ jobs:
           evict-old-files: 1d
           max-size: "1G"
 
-      - name: Tune ccache for ROCm
-        run: |
-          # since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
-          ccache --set-config=depend_mode=true
-
       # - name: Cache ROCm Installation
       #   id: cache-rocm
       #   uses: actions/cache@v5
@@ -1211,12 +1204,12 @@ jobs:
           - cuda: '12.4'
             arch: x64
             defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
-          - cuda: '13.5'
+          - cuda: '13.4'
             arch: x64
-            defines: ''
-          - cuda: '13.5'
+            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3'
+          - cuda: '13.4'
             arch: arm64
-            defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
+            defines: '-DGGML_CUDA_CCCL_VERSION=v3.4.3 -DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
 
     steps:
       - name: Clone
@@ -1552,8 +1545,6 @@ jobs:
           # builds hit.
           ccache --set-config=compiler_check=content
           ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
-          # since ROCm 10.1, clang -E output has no GPU code, so the preprocessor fallback can reuse objects with old GPU code
-          ccache --set-config=depend_mode=true
 
       - name: Dependencies
         id: depends
@@ -1628,60 +1619,6 @@ jobs:
         with:
           key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
 
-  ubuntu-26-zdnn:
-    needs: [check-release, ui-build]
-    if: ${{ needs.check-release.outputs.should_release == 'true' }}
-
-    runs-on: ubuntu-26.04-s390x
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Download UI build
-        uses: actions/download-artifact@v8
-        with:
-          name: llama-ui.zip
-          path: tools/ui/dist
-
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential libssl-dev libzdnn-dev
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DGGML_BACKEND_DL=ON \
-            -DGGML_NATIVE=OFF \
-            -DGGML_VXE=ON \
-            -DGGML_ZDNN=ON \
-            -DCMAKE_C_FLAGS="-march=arch15" \
-            -DCMAKE_CXX_FLAGS="-march=arch15" \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            ${{ env.CMAKE_ARGS }}
-          cmake --build build --config Release -j $(nproc)
-
-      - name: Pack artifacts
-        id: pack_artifacts
-        run: |
-          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v7
-        with:
-          path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz
-          archive: false
-
   ios-xcode:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}

.github/workflows/server-speculative.yml

@@ -0,0 +1,65 @@
+name: Server (speculative decoding)
+
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: 'Commit SHA1 to build'
+        required: false
+        type: string
+
+cache-mode: none
+permissions:
+  contents: read
+
+env:
+  LLAMA_ARG_LOG_COLORS: 1
+  LLAMA_ARG_LOG_PREFIX: 1
+  LLAMA_ARG_LOG_TIMESTAMPS: 1
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  server-speculative:
+    runs-on: [self-hosted, Linux, gb10, perf]
+    container: "nvidia/cuda:13.4.3-devel-ubuntu26.04"
+    timeout-minutes: 60
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y cmake libssl-dev python3 python3-venv python3-pip
+
+      - name: Build
+        run: |
+          cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+          cmake --build build --config Release -j $(nproc) --target llama-server
+
+      - name: Python setup
+        run: |
+          cd tools/server/tests
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+
+      - name: Test speculative decoding
+        env:
+          SPEC_METHODS: compact
+          SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
+          N_GPU_LAYERS: 99
+        run: |
+          cd tools/server/tests
+          source venv/bin/activate
+          PYTEST_WORKERS=1 ./tests.sh -s -v unit/test_spec_methods.py

CMakeLists.txt

@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)
 
 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 9)
+set(LLAMA_VERSION_MINOR 7)
 set(LLAMA_VERSION_PATCH 0)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
 
@@ -103,7 +103,7 @@ endif()
 
 # subprocess spawning isn't a supported/sandbox-friendly operation on mobile OSes or in WASM
 if (CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android" OR ANDROID
-        OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN OR CMAKE_SYSTEM_NAME STREQUAL "WASI")
+        OR CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR EMSCRIPTEN)
     set(LLAMA_SUBPROCESS_DEFAULT OFF)
 else()
     set(LLAMA_SUBPROCESS_DEFAULT ON)
@@ -150,7 +150,6 @@ option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured
 # Required for relocatable CMake package
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/win-app.cmake)
 
 if (NOT DEFINED LLAMA_BUILD_NUMBER)
     set(LLAMA_BUILD_NUMBER        ${BUILD_NUMBER})
@@ -270,8 +269,6 @@ if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
     add_subdirectory(tools/mtmd)
 endif()
 
-llama_apply_app_manifests()
-
 #
 # install
 #

CODEOWNERS

@@ -2,29 +2,20 @@
 # multiple collaborators per item can be specified
 #
 # ggml-org/ci               : CISC, danbev, ggerganov, netrunnereve, ngxson, taronaeo
-# ggml-org/ggml-base        : aendk, fairydreaming, ggerganov
 # ggml-org/ggml-cann        : hipudding
-# ggml-org/ggml-cpu         : angt, chaxu01, ggerganov, max-krasnyansky, taronaeo
-# ggml-org/ggml-cuda        : am17an, gaugarg-nv, IMbackK, JohannesGaessler, ORippler, thelittlefireman, ynankani
-# ggml-org/ggml-et          : marty1885
+# ggml-org/ggml-cuda        : JohannesGaessler, am17an, IMbackK, ORippler
 # ggml-org/ggml-hexagon     : lhez, max-krasnyansky
-# ggml-org/ggml-hip         : IMbackK, thelittlefireman
-# ggml-org/ggml-metal       : forforever73, ggerganov, jhen0409, masterFoad
+# ggml-org/ggml-metal       : ggerganov
 # ggml-org/ggml-opencl      : lhez, max-krasnyansky
-# ggml-org/ggml-openvino    : ravi9, wine99
-# ggml-org/ggml-riscv       : xctan
-# ggml-org/ggml-rpc         : am17an, rgerganov
-# ggml-org/ggml-sycl        : arthw, Titaniumtown
+# ggml-org/ggml-rpc         : rgerganov
+# ggml-org/ggml-sycl        : arthw
 # ggml-org/ggml-vulkan      : 0cc4m, jeffbolznv
 # ggml-org/ggml-webgpu      : reeselevine, yomaytk
-# ggml-org/ggml-zdnn        : AlekseiNikiforovIBM, Andreas-Krebbel, taronaeo
-# ggml-org/llama-base       : am17an, CISC, ggerganov, ngxson, sanmai, ServeurpersoCom
-# ggml-org/llama-common     : aldehir, angt, bartowski1182, danbev, ggerganov, ngxson, nikwen, pwilkin, ruixiang63, sanmai
-# ggml-org/llama-examples   : ggerganov
-# ggml-org/llama-models     : CISC, danbev, ggerganov, ngxson
+# ggml-org/ggml-zdnn        : taronaeo
+# ggml-org/llama-common     : ggerganov, aldehir, angt, danbev, ngxson, pwilkin
 # ggml-org/llama-mtmd       : ngxson
-# ggml-org/llama-server     : allozaur, angt, ggerganov, ngxson, ServeurpersoCom
-# ggml-org/llama-ui         : allozaur
+# ggml-org/llama-server     : ggerganov, ngxson, allozaur, angt, ServeurpersoCom
+# ggml-org/llama-ui           : allozaur
 
 /.devops/*.Dockerfile                   @ngxson
 /.github/actions/                       @ggml-org/ci
@@ -38,12 +29,28 @@
 /conversion/                            @CISC
 /convert_*.py                           @CISC
 /docs/backend/snapdragon/               @ggml-org/ggml-hexagon
-/examples/                              @ggml-org/llama-examples
+/examples/batched.swift/                @ggerganov
+/examples/batched/                      @ggerganov
+/examples/convert-llama2c-to-ggml/      @ggerganov
 /examples/debug/                        @danbev @pwilkin
+/examples/deprecation-warning/          @ggerganov
 /examples/diffusion/                    @am17an
+/examples/embedding/                    @ggerganov
+/examples/eval-callback/                @ggerganov
+/examples/export-docs/                  @ggerganov
+/examples/gen-docs/                     @ggerganov
+/examples/gguf/                         @ggerganov
 /examples/llama.android/                @ggerganov @hanyin-arm @naco-siren
+/examples/llama.swiftui/                @ggerganov
+/examples/llama.vim                     @ggerganov
+/examples/lookahead/                    @ggerganov
 /examples/lookup/                       @JohannesGaessler
 /examples/model-conversion/             @danbev
+/examples/parallel/                     @ggerganov
+/examples/passkey/                      @ggerganov
+/examples/retrieval/                    @ggerganov
+/examples/speculative-simple/           @ggerganov
+/examples/speculative/                  @ggerganov
 /ggml/cmake/                            @ggerganov
 /ggml/include/                          @ggerganov
 /ggml/src/ggml-backend-meta.cpp         @JohannesGaessler
@@ -55,7 +62,7 @@
 /ggml/src/ggml-cuda/                    @ggml-org/ggml-cuda
 /ggml/src/ggml-cuda/vendors/hip.h       @IMbackK
 /ggml/src/ggml-hexagon/                 @ggml-org/ggml-hexagon
-/ggml/src/ggml-hip/                     @ggml-org/ggml-hip
+/ggml/src/ggml-hip/                     @IMbackK
 /ggml/src/ggml-et/                      @marty1885
 /ggml/src/ggml-impl.h                   @ggerganov
 /ggml/src/ggml-metal/                   @ggml-org/ggml-metal
@@ -69,7 +76,7 @@
 /ggml/src/ggml-virtgpu/                 @kpouget
 /ggml/src/ggml-vulkan/                  @ggml-org/ggml-vulkan
 /ggml/src/ggml-webgpu/                  @ggml-org/ggml-webgpu
-/ggml/src/ggml-zdnn/                    @ggml-org/ggml-zdnn
+/ggml/src/ggml-zdnn/                    @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
 /ggml/src/ggml-zendnn/                  @avinashcpandey @Jiten1parmar
 /ggml/src/ggml.c                        @ggerganov
 /ggml/src/ggml.cpp                      @ggerganov
@@ -80,14 +87,14 @@
 /scripts/get*                           @ggerganov
 /scripts/sync*                          @ggerganov
 /scripts/snapdragon/                    @ggml-org/ggml-hexagon
-/src/                                   @ggml-org/llama-base
+/src/                                   @ggerganov
 /src/llama-adapter.*                    @CISC
 /src/llama-arch.*                       @CISC
 /src/llama-chat.*                       @ngxson
 /src/llama-graph.*                      @CISC
 /src/llama-model.*                      @CISC
 /src/llama-vocab.*                      @CISC
-/src/models/                            @ggml-org/llama-models
+/src/models/                            @CISC
 /tests/                                 @ggerganov
 /tests/test-chat.*                      @pwilkin
 /tests/test-server-component.cpp        @ggml-org/llama-server

README.md

@@ -13,7 +13,7 @@
 [![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
 [![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
 
-[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
+[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
 
 </div>
 
@@ -79,7 +79,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 | --- | --- |
 | [BLAS](docs/build.md#blas-build) | All |
 | [BLIS](docs/backend/BLIS.md) | All |
-| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
+| [CANN](docs/build.md#cann) | Ascend NPU |
 | [CUDA](docs/build.md#cuda) | Nvidia GPU |
 | [HIP](docs/build.md#hip) | AMD GPU |
 | [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |

ci/run.sh

@@ -185,28 +185,18 @@ fi
 
 ## helpers
 
-# download a file unless already cached in $MNT
+# download a file if it does not exist or if it is outdated
 function gg_wget {
     local out=$1
     local url=$2
 
     local cwd=`pwd`
-    local name=`basename $url`
-    local tmp=.gg-wget.$$.tmp
 
     mkdir -p $out
     cd $out
 
-    # cache first: HF artifacts are static, HF sends no Last-Modified for -N
-    if [ ! -s $name ]; then
-        # publish with mv: concurrent runners never read partial files
-        rm -rf $tmp && mkdir $tmp
-        (cd $tmp && wget -nv $url)
-        if [ -s $tmp/$name ]; then
-            mv -f $tmp/$name $name
-        fi
-        rm -rf $tmp
-    fi
+    # should not re-download if file is the same
+    wget -nv -c -N $url
 
     cd $cwd
 }
@@ -264,7 +254,7 @@ function gg_run_ctest_release {
     (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
 
     if [ -z ${GG_BUILD_LOW_PERF} ]; then
-        (time ctest -C Release --output-on-failure -L main ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
+        (time ctest -C Release --output-on-failure -L 'main|python' ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
     else
         (time ctest -C Release --output-on-failure -L main -E test-opt ${CTEST_EXTRA}) 2>&1 | tee -a $OUT/${ci}-ctest.log
     fi
@@ -296,6 +286,25 @@ function gg_run_test_llama_archs_tensor_split {
     set +e
 }
 
+# test_llama_archs_models
+
+function gg_run_test_llama_archs_models {
+    cd ${SRC}
+
+    set -e
+
+    # TODO: fix and re-enable `test-llama-archs` on OpenVINO
+    # TODO: the `test-llama-archs` currently does not build on Windows, so we check if the binary exists
+    if [ -z ${GG_BUILD_OPENVINO} ] && [ -f ./build-ci-release/bin/test-llama-archs ]; then
+        rm -rf build-ci-models && mkdir -p build-ci-models
+
+        # generate the dummy models used by the model-dependent tests
+        ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
+    fi
+
+    set +e
+}
+
 # test_scripts
 
 function gg_run_test_scripts {
@@ -303,8 +312,8 @@ function gg_run_test_scripts {
 
     set -e
 
-    (cd ./tools/gguf-split && time bash tests.sh "$SRC/build-ci-release/bin" "$OUT/scratch") 2>&1 | tee -a $OUT/${ci}-scripts.log
-    (cd ./tools/quantize   && time bash tests.sh "$SRC/build-ci-release/bin" "$OUT/scratch") 2>&1 | tee -a $OUT/${ci}-scripts.log
+    (cd ./tools/gguf-split && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
+    (cd ./tools/quantize   && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
 
     set +e
 }
@@ -365,14 +374,10 @@ function gg_run_qwen3_0_6b {
 
 
     gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip
-
-    rm -rf $OUT/scratch/qwen3-0.6B $OUT/scratch/wikitext
-    mkdir -p $OUT/scratch/qwen3-0.6B
-    unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d $OUT/scratch/wikitext/
+    unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
 
     path_models="../models-mnt/qwen3/0.6B"
-    path_local="$OUT/scratch/qwen3-0.6B"
-    path_wiki="$OUT/scratch/wikitext/wikitext-2-raw"
+    path_wiki="../models-mnt/wikitext/wikitext-2-raw"
 
     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
 
@@ -381,15 +386,72 @@ function gg_run_qwen3_0_6b {
     (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
     (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
 
-    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf  --outtype f16
-    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-bf16.gguf --outtype bf16
+    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf  --outtype f16
+    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-bf16.gguf --outtype bf16
 
-    model_f16="${path_local}/ggml-model-f16.gguf"
-    model_bf16="${path_local}/ggml-model-bf16.gguf"
-    model_q4_0="${path_local}/ggml-model-q4_0.gguf"
+    model_f16="${path_models}/ggml-model-f16.gguf"
+    model_bf16="${path_models}/ggml-model-bf16.gguf"
+    model_q8_0="${path_models}/ggml-model-q8_0.gguf"
+    model_q4_0="${path_models}/ggml-model-q4_0.gguf"
+    model_q4_1="${path_models}/ggml-model-q4_1.gguf"
+    model_q5_0="${path_models}/ggml-model-q5_0.gguf"
+    model_q5_1="${path_models}/ggml-model-q5_1.gguf"
+    model_q2_k="${path_models}/ggml-model-q2_k.gguf"
+    model_q3_k="${path_models}/ggml-model-q3_k.gguf"
+    model_q4_k="${path_models}/ggml-model-q4_k.gguf"
+    model_q5_k="${path_models}/ggml-model-q5_k.gguf"
+    model_q6_k="${path_models}/ggml-model-q6_k.gguf"
 
     wiki_test="${path_wiki}/wiki.test.raw"
 
+    ./bin/llama-quantize ${model_bf16} ${model_q8_0} q8_0 $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q4_0} q4_0 $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q4_1} q4_1 $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q5_0} q5_0 $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q5_1} q5_1 $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q2_k} q2_k $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q3_k} q3_k $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q4_k} q4_k $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q5_k} q5_k $(nproc)
+    ./bin/llama-quantize ${model_bf16} ${model_q6_k} q6_k $(nproc)
+
+    (time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
+
+    (time ./bin/llama-completion -no-cnv --model ${model_f16}  -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
+    (time ./bin/llama-completion -no-cnv --model ${model_bf16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q8_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q4_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q4_1} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q5_0} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q5_1} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q2_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q3_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q4_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q5_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
+    (time ./bin/llama-completion -no-cnv --model ${model_q6_k} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
+
+    (time ./bin/llama-perplexity --model ${model_f16}  -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
+    if [ -z ${GG_BUILD_NO_BF16} ]; then
+        (time ./bin/llama-perplexity --model ${model_bf16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
+    fi
+    (time ./bin/llama-perplexity --model ${model_q8_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
+    (time ./bin/llama-perplexity --model ${model_q4_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
+    (time ./bin/llama-perplexity --model ${model_q4_1} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
+    (time ./bin/llama-perplexity --model ${model_q5_0} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
+    (time ./bin/llama-perplexity --model ${model_q5_1} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
+    (time ./bin/llama-perplexity --model ${model_q2_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
+    (time ./bin/llama-perplexity --model ${model_q3_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
+    (time ./bin/llama-perplexity --model ${model_q4_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
+    (time ./bin/llama-perplexity --model ${model_q5_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
+    (time ./bin/llama-perplexity --model ${model_q6_k} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
+
+    (time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
+
+    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off                ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
+
     function check_ppl {
         qnt="$1"
         ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
@@ -403,52 +465,20 @@ function gg_run_qwen3_0_6b {
         return 0
     }
 
-    for qnt in q8_0 q4_0 q4_1 q5_0 q5_1 q2_k q3_k q4_k q5_k q6_k; do
-        model="${path_local}/ggml-model-${qnt}.gguf"
-
-        ./bin/llama-quantize ${model_bf16} ${model} ${qnt} $(nproc)
-
-        (time ./bin/llama-completion -no-cnv --model ${model} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-${qnt}.log
-        (time ./bin/llama-perplexity --model ${model} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-${qnt}.log
-
-        # note: ppl > 20.0 for q2_k with this model
-        if [ "${qnt}" != "q2_k" ]; then
-            check_ppl "${qnt}" "$(cat $OUT/${ci}-tg-${qnt}.log | grep "^\[1\]")"
-        fi
-
-        if [ "${qnt}" = "q4_0" ]; then
-            # q4_0 is kept for other tests
-            q4_0="${path_models}/ggml-model-q4_0.gguf"
-            ln ${model} "$q4_0" 2>/dev/null || { cp ${model} $q4_0.tmp.$$ && mv -f $q4_0.tmp.$$ $q4_0; }
-        else
-            rm -f ${model}
-        fi
-    done
-
-    (time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
-
-    (time ./bin/llama-completion -no-cnv --model ${model_f16}  -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
-    (time ./bin/llama-completion -no-cnv --model ${model_bf16} -ngl 99 -c 1024 -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
-
-    (time ./bin/llama-perplexity --model ${model_f16}  -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
-    if [ -z ${GG_BUILD_NO_BF16} ]; then
-        (time ./bin/llama-perplexity --model ${model_bf16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-tg-bf16.log
-    fi
-
-    (time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
-
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa off                ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 10 -c 1024 -fa on  --no-op-offload) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-    (time ./bin/test-llama-context --model ${model_q4_0} -ngl 99 -c 1024 -fa on                 ) 2>&1 | tee -a $OUT/${ci}-test-llama-context.log
-
     check_ppl "f16"  "$(cat $OUT/${ci}-tg-f16.log  | grep "^\[1\]")"
     if [ -z ${GG_BUILD_NO_BF16} ]; then
         check_ppl "bf16" "$(cat $OUT/${ci}-tg-bf16.log | grep "^\[1\]")"
     fi
-
-    # scratch is rebuilt on every run; q4_0 is already published to $MNT
-    rm -rf $OUT/scratch/qwen3-0.6B $OUT/scratch/wikitext
+    check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")"
+    check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")"
+    check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")"
+    check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")"
+    check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")"
+   #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" # note: ppl > 20.0 for this quant and model
+    check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")"
+    check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")"
+    check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")"
+    check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")"
 
     set +e
 }
@@ -471,8 +501,6 @@ function gg_run_embd_bge_small {
     gg_wget models-mnt/bge-small/1_Pooling https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json
 
     path_models="../models-mnt/bge-small"
-    path_local="$OUT/scratch/bge-small"
-    rm -rf $path_local && mkdir -p $path_local
 
     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
 
@@ -481,10 +509,10 @@ function gg_run_embd_bge_small {
     (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
     (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
 
-    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf
+    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf
 
-    model_f16="${path_local}/ggml-model-f16.gguf"
-    model_q8_0="${path_local}/ggml-model-q8_0.gguf"
+    model_f16="${path_models}/ggml-model-f16.gguf"
+    model_q8_0="${path_models}/ggml-model-q8_0.gguf"
 
     ./bin/llama-quantize ${model_f16} ${model_q8_0} q8_0
 
@@ -493,8 +521,6 @@ function gg_run_embd_bge_small {
     (time ./bin/llama-embedding --model ${model_f16}  -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
     (time ./bin/llama-embedding --model ${model_q8_0} -p "I believe the meaning of life is" -ngl 99 -c 0 --no-op-offload) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
 
-    rm -rf $path_local
-
     set +e
 }
 
@@ -511,8 +537,6 @@ function gg_run_rerank_tiny {
     gg_wget models-mnt/rerank-tiny/ https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/raw/main/vocab.json
 
     path_models="../models-mnt/rerank-tiny"
-    path_local="$OUT/scratch/rerank-tiny"
-    rm -rf $path_local && mkdir -p $path_local
 
     rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
 
@@ -521,9 +545,9 @@ function gg_run_rerank_tiny {
     (cmake -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
     (time cmake --build . --config Release -j$(nproc)) 2>&1 | tee -a $OUT/${ci}-make.log
 
-    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_local}/ggml-model-f16.gguf
+    python3 ../convert_hf_to_gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf
 
-    model_f16="${path_local}/ggml-model-f16.gguf"
+    model_f16="${path_models}/ggml-model-f16.gguf"
 
     (time ./bin/llama-fit-params --model ${model_f16} 2>&1 | tee -a $OUT/${ci}-fp-f16.log)
 
@@ -553,8 +577,6 @@ function gg_run_rerank_tiny {
     check_score "rerank score 1" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 1")" "0.00" "0.05" | tee -a $OUT/${ci}-rk-f16.log
     check_score "rerank score 2" "$(cat $OUT/${ci}-rk-f16.log | grep "rerank score 2")" "0.10" "0.30" | tee -a $OUT/${ci}-rk-f16.log
 
-    rm -rf $path_local
-
     set +e
 }
 
@@ -661,11 +683,11 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then
     ln -sfn ${mnt_models} ${SRC}/models-mnt
 
     # Create a fresh python3 venv and enter it
-    if ! python3 -m venv "$OUT/venv"; then
-        echo "Error: Failed to create Python virtual environment at $OUT/venv."
+    if ! python3 -m venv "$MNT/venv"; then
+        echo "Error: Failed to create Python virtual environment at $MNT/venv."
         exit 1
     fi
-    source "$OUT/venv/bin/activate"
+    source "$MNT/venv/bin/activate"
 
     pip install -r ${SRC}/requirements.txt --disable-pip-version-check
     pip install --editable gguf-py --disable-pip-version-check
@@ -678,6 +700,7 @@ test $ret -eq 0 && gg_run ctest_release
 
 test $ret -eq 0 && gg_run test_backend_ops
 
+test $ret -eq 0 && gg_run test_llama_archs_models
 test $ret -eq 0 && gg_run test_llama_archs_tensor_split
 
 if [ -z ${GG_BUILD_LOW_PERF} ]; then

cmake/win-app.cmake

@@ -1,66 +0,0 @@
-set(LLAMA_WIN_APP_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/../common/win-app.manifest")
-
-if(WIN32)
-    if(MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
-        set(LLAMA_WIN_APP_MSVC_ABI TRUE)
-    else()
-        set(LLAMA_WIN_APP_MSVC_ABI FALSE)
-    endif()
-
-    if(NOT LLAMA_WIN_APP_MSVC_ABI)
-        if(CMAKE_VERSION VERSION_LESS 3.21)
-            message(FATAL_ERROR "CMake 3.21 or newer is required to build for Windows with the GNU ABI")
-        endif()
-        enable_language(RC)
-        set(LLAMA_WIN_APP_MANIFEST_RC "${CMAKE_CURRENT_BINARY_DIR}/win-app-manifest.rc")
-        file(CONFIGURE OUTPUT "${LLAMA_WIN_APP_MANIFEST_RC}" CONTENT "1 24 \"@LLAMA_WIN_APP_MANIFEST@\"\n" @ONLY)
-        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
-    endif()
-endif()
-
-function(llama_add_app_manifest target)
-    if(NOT WIN32)
-        return()
-    endif()
-    get_target_property(_aliased ${target} ALIASED_TARGET)
-    if(_aliased)
-        return()
-    endif()
-    get_target_property(_type ${target} TYPE)
-    if(NOT _type STREQUAL "EXECUTABLE")
-        return()
-    endif()
-    if(LLAMA_WIN_APP_MSVC_ABI)
-        target_sources(${target} PRIVATE "${LLAMA_WIN_APP_MANIFEST}")
-    else()
-        if(NOT TARGET llama-win-app-manifest-res)
-            add_library(llama-win-app-manifest-res OBJECT ${LLAMA_WIN_APP_MANIFEST_RC})
-            set_source_files_properties(${LLAMA_WIN_APP_MANIFEST_RC} PROPERTIES OBJECT_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
-        endif()
-        target_link_libraries(${target} PRIVATE "$<TARGET_OBJECTS:llama-win-app-manifest-res>")
-    endif()
-endfunction()
-
-function(llama_apply_app_manifests_dir dir)
-    get_property(_targets DIRECTORY "${dir}" PROPERTY BUILDSYSTEM_TARGETS)
-    foreach(_target IN LISTS _targets)
-        llama_add_app_manifest(${_target})
-    endforeach()
-    get_property(_subdirs DIRECTORY "${dir}" PROPERTY SUBDIRECTORIES)
-    foreach(_subdir IN LISTS _subdirs)
-        llama_apply_app_manifests_dir("${_subdir}")
-    endforeach()
-endfunction()
-
-function(llama_apply_app_manifests)
-    if(NOT WIN32)
-        return()
-    endif()
-    get_property(_subdirs DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY SUBDIRECTORIES)
-    foreach(_dir IN LISTS _subdirs)
-        get_filename_component(_name "${_dir}" NAME)
-        if(_name STREQUAL "app" OR _name STREQUAL "examples" OR _name STREQUAL "pocs" OR _name STREQUAL "tests" OR _name STREQUAL "tools")
-            llama_apply_app_manifests_dir("${_dir}")
-        endif()
-    endforeach()
-endfunction()

common/CMakeLists.txt

@@ -78,8 +78,6 @@ add_library(${TARGET}
     download.h
     fit.cpp
     fit.h
-    file-lock.cpp
-    file-lock.h
     hf-cache.cpp
     hf-cache.h
     http.h
@@ -186,8 +184,3 @@ if (LLAMA_LLGUIDANCE)
 endif()
 
 target_link_libraries(${TARGET} PUBLIC llama Threads::Threads)
-
-if (CMAKE_SYSTEM_NAME MATCHES "AIX")
-    # AIX's flock() function comes from libbsd.a
-    target_link_libraries(${TARGET} PRIVATE -lbsd)
-endif()

common/arg.cpp

@@ -4,7 +4,6 @@
 #include "chat.h"
 #include "common.h"
 #include "download.h"
-#include "http.h"
 #include "json-schema-to-grammar.h"
 #include "json.h"
 #include "llama.h"
@@ -13,6 +12,16 @@
 #include "speculative.h"
 #include "preset.h"
 
+// fix problem with std::min and std::max
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#   define NOMINMAX
+#endif
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
 #include <algorithm>
 #include <cinttypes>
 #include <climits>
@@ -29,6 +38,20 @@
 #include <thread> // for hardware_concurrency
 #include <vector>
 
+#ifndef __EMSCRIPTEN__
+#ifdef __linux__
+#include <linux/limits.h>
+#elif defined(_WIN32)
+#   if !defined(PATH_MAX)
+#   define PATH_MAX MAX_PATH
+#   endif
+#elif defined(_AIX)
+#include <sys/limits.h>
+#else
+#include <sys/syslimits.h>
+#endif
+#endif
+
 #define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
 
 using json = common_json;
@@ -328,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
 static std::string get_default_local_path(const std::string & url) {
     auto f = string_split<std::string>(url, '#').front();
     f = string_split<std::string>(f, '?').front();
-    return fs_get_cache_file(string_split<std::string>(f, '/').back()).string();
+    return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
 }
 
 static bool spec_types_is_default(const common_params & params) {
@@ -400,26 +423,7 @@ bool common_models_handler_is_preset_repo(const common_models_handler & handler)
     return !handler.plan.preset.url.empty();
 }
 
-static common_download_opts download_opts_for_url(const common_download_opts & opts, const std::string & url) {
-    common_download_opts result = opts;
-    try {
-        const auto hf    = common_http_parse_url(common_get_model_endpoint());
-        const auto parts = common_http_parse_url(url);
-
-        if (parts.scheme == hf.scheme &&
-            parts.host   == hf.host &&
-            parts.port   == hf.port &&
-            string_starts_with(result.bearer_token, "hf_")) {
-            return result;
-        }
-    } catch (const std::exception &) {
-    }
-    result.bearer_token.clear();
-    return result;
-}
-
 static std::vector<common_download_task> build_url_tasks(const common_params_model & model, common_download_opts opts) {
-    opts = download_opts_for_url(opts, model.url);
     auto parts = common_download_get_all_parts(model.url);
     std::vector<common_download_task> tasks;
 
@@ -466,14 +470,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
     auto opts = handler.opts; // copy
     opts.callback = callback;
 
-    // another process may hold the cache lock during finalization:
-    // let the download callback cancel that wait too
-    auto finalize = [callback](const hf_cache::hf_file & f) {
-        return hf_cache::finalize_file(f, [callback]() {
-            return !(callback && callback->is_cancelled());
-        });
-    };
-
     // handle plain "url" if needed
     auto handle_url = [&](common_params_model & model) {
         if (!model.url.empty()) {
@@ -508,7 +504,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         common_download_task task;
         task.url        = params.mmproj.url;
         task.local_path = params.mmproj.path;
-        task.opts       = download_opts_for_url(opts, task.url);
+        task.opts       = opts;
         tasks.push_back(task);
     }
     bool had_spec_url = false;
@@ -516,13 +512,13 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         common_download_task task;
         task.url        = params.speculative.draft.mparams.url;
         task.local_path = params.speculative.draft.mparams.path;
-        task.opts       = download_opts_for_url(opts, task.url);
+        task.opts       = opts;
         tasks.push_back(task);
         had_spec_url = true;
     }
 
     // handle hf_plan tasks
-    auto add_tasks = [&opts, &tasks, &finalize](const hf_cache::hf_files  & model_files,
+    auto add_tasks = [&opts, &tasks](const hf_cache::hf_files  & model_files,
                                     const hf_cache::hf_file    & primary,
                                     common_params_model        & model) {
         for (size_t i = 0; i < model_files.size(); ++i) {
@@ -531,9 +527,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
             tasks.emplace_back(model_file, opts, [&, is_primary]() {
                 if (is_primary) {
                     // the primary file is the first split (00001-of), use it as model path
-                    model.path = finalize(model_file);
+                    model.path = hf_cache::finalize_file(model_file);
                 } else {
-                    finalize(model_file);
+                    hf_cache::finalize_file(model_file);
                 }
             });
         }
@@ -585,9 +581,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan_spec.mtp, opts, [&]() {
             // only use the discovered MTP head when no draft path is set yet
             if (params.speculative.draft.mparams.path.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan_spec.mtp);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.mtp);
             } else {
-                finalize(plan_spec.mtp);
+                hf_cache::finalize_file(plan_spec.mtp);
             }
         });
     }
@@ -595,9 +591,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan_spec.dflash, opts, [&]() {
             // only use the discovered DFlash sidecar when no draft path is set yet
             if (params.speculative.draft.mparams.path.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan_spec.dflash);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dflash);
             } else {
-                finalize(plan_spec.dflash);
+                hf_cache::finalize_file(plan_spec.dflash);
             }
         });
     }
@@ -605,9 +601,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan_spec.eagle3, opts, [&]() {
             // only use the discovered Eagle3 sidecar when no draft path is set yet
             if (params.speculative.draft.mparams.path.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan_spec.eagle3);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.eagle3);
             } else {
-                finalize(plan_spec.eagle3);
+                hf_cache::finalize_file(plan_spec.eagle3);
             }
         });
     }
@@ -615,9 +611,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan_spec.dspark, opts, [&]() {
             // only use the discovered DSpark sidecar when no draft path is set yet
             if (params.speculative.draft.mparams.path.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan_spec.dspark);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark);
             } else {
-                finalize(plan_spec.dspark);
+                hf_cache::finalize_file(plan_spec.dspark);
             }
         });
     }
@@ -638,16 +634,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params
     }
     if (!plan.mmproj.local_path.empty()) {
         tasks.emplace_back(plan.mmproj, opts, [&]() {
-            params.mmproj.path = finalize(plan.mmproj);
+            params.mmproj.path = hf_cache::finalize_file(plan.mmproj);
         });
     }
     if (!plan.mtp.local_path.empty() && !had_spec_url) {
         tasks.emplace_back(plan.mtp, opts, [&]() {
             // only fall back to the discovered MTP head when no draft was explicitly provided
             if (params.speculative.draft.mparams.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan.mtp);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.mtp);
             } else {
-                finalize(plan.mtp);
+                hf_cache::finalize_file(plan.mtp);
             }
         });
     }
@@ -655,9 +651,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan.dflash, opts, [&]() {
             // only fall back to the discovered DFlash sidecar when no draft was explicitly provided
             if (params.speculative.draft.mparams.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan.dflash);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dflash);
             } else {
-                finalize(plan.dflash);
+                hf_cache::finalize_file(plan.dflash);
             }
         });
     }
@@ -665,9 +661,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan.eagle3, opts, [&]() {
             // only fall back to the discovered Eagle3 sidecar when no draft was explicitly provided
             if (params.speculative.draft.mparams.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan.eagle3);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.eagle3);
             } else {
-                finalize(plan.eagle3);
+                hf_cache::finalize_file(plan.eagle3);
             }
         });
     }
@@ -675,9 +671,9 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan.dspark, opts, [&]() {
             // only fall back to the discovered DSpark sidecar when no draft was explicitly provided
             if (params.speculative.draft.mparams.empty()) {
-                params.speculative.draft.mparams.path = finalize(plan.dspark);
+                params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark);
             } else {
-                finalize(plan.dspark);
+                hf_cache::finalize_file(plan.dspark);
             }
         });
     }
@@ -685,7 +681,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params
         tasks.emplace_back(plan.preset, opts, [&]() {
             // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file
             params.models_preset_hf = params.model.hf_repo; // only for showing a warning
-            params.models_preset    = finalize(plan.preset);
+            params.models_preset    = hf_cache::finalize_file(plan.preset);
             // clear the model so the server starts in router mode
             params.model.path.clear();
             params.model.hf_repo.clear();
@@ -730,7 +726,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
     std::vector<std::filesystem::path> paths;
 
 #if defined(_WIN32)
-    const std::filesystem::path program_data = common_get_env("PROGRAMDATA");
+    const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA");
     if (!program_data.empty()) {
         paths.push_back(program_data / "llama.cpp" / "config.ini");
     }
@@ -758,7 +754,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
     common_preset_context ctx(ex);
     ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
     for (const auto & path : found) {
-        LOG_INF("using config file: %s\n", path.string().c_str());
+        LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
         common_preset global;
         common_presets presets = ctx.load_from_ini(path, global);
         global.apply_to_params(params);
@@ -920,7 +916,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
         // TODO @ngxson : maybe show a list of available models in CLI in this case
         bool can_skip_model = params.usage || params.completion || !params.server_base.empty();
         if (!can_skip_model && params.model.path.empty()) {
-            throw std::invalid_argument("error: --model or -hf is required\n");
+            throw std::invalid_argument("error: --model is required\n");
         }
     }
 
@@ -1255,7 +1251,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
     return true;
 }
 
+#ifdef _WIN32
+struct utf8_argv {
+    std::vector<std::string> buf;
+    std::vector<char*> ptrs;
+};
+
+static utf8_argv make_utf8_argv() {
+    utf8_argv out;
+    int wargc = 0;
+    LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
+    if (!wargv) return out;
+
+    out.buf.reserve(wargc);
+    for (int i = 0; i < wargc; ++i) {
+        int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr);
+        if (n <= 0) { out.buf.emplace_back(); continue; }
+        auto& s = out.buf.emplace_back();
+        s.resize(static_cast<size_t>(n - 1));
+        (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr);
+    }
+    LocalFree(wargv);
+
+    out.ptrs.reserve(out.buf.size() + 1);
+    for (auto& s : out.buf) out.ptrs.push_back(s.data());
+    out.ptrs.push_back(nullptr);
+    return out;
+}
+#endif
+
 bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {
+#ifdef _WIN32
+    auto utf8 = make_utf8_argv();
+    // repair argv only when it matches the process command line
+    if (static_cast<int>(utf8.buf.size()) == argc) {
+        argv = utf8.ptrs.data();
+    }
+#endif
+
     auto ctx_arg = common_params_parser_init(params, ex, print_usage);
     const common_params params_org = ctx_arg.params; // the example can modify the default params
 
@@ -3667,13 +3700,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.models_max = value;
         }
     ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MAX"));
-    add_opt(common_arg(
-        {"--models-memory-margin"}, "N",
-        string_format("for router server, MiB of device memory to keep free per device, unload LRU models until a new one fits (default: %d, 0 = disabled)", params.models_memory_margin),
-        [](common_params & params, int value) {
-            params.models_memory_margin = value;
-        }
-    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MODELS_MEMORY_MARGIN"));
     add_opt(common_arg(
         {"--models-autoload"},
         {"--no-models-autoload"},
@@ -4471,14 +4497,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.tts_lang = value;
         }
     ).set_examples({LLAMA_EXAMPLE_TTS}));
-    add_opt(common_arg(
-        {"--tts-voice"}, "VOICE",
-        "voice for audio generation (model-dependent)\n"
-        "see tts/README.md for per-model usage notes",
-        [](common_params & params, const std::string & value) {
-            params.tts_voice = value;
-        }
-    ).set_examples({LLAMA_EXAMPLE_TTS}));
     add_opt(common_arg(
         {"--tts-speaker-file"}, "FNAME",
         "speaker file path for audio generation",

common/chat.cpp

@@ -190,17 +190,10 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
         } else {
             auto & parts = jmsg["content"] = json::array();
             for (const auto & part : content_parts) {
-                if (part.type == "tool_reference") {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"name", part.name},
-                    });
-                } else {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"text", part.text},
-                    });
-                }
+                parts.push_back({
+                    {"type", part.type},
+                    {"text", part.text},
+                });
             }
         }
     } else {
@@ -316,7 +309,6 @@ struct common_chat_templates {
     bool has_explicit_template;  // Model had builtin template or template overridden was specified.
     std::unique_ptr<common_chat_template> template_default;  // always set (defaults to chatml)
     std::unique_ptr<common_chat_template> template_tool_use;
-    std::unique_ptr<common_chat_template> template_tts;  // the prompt of a text to speak
 };
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -378,16 +370,12 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
                             throw std::invalid_argument("Missing content part type: " + part.dump());
                         }
                         const auto & type = part.at("type");
-                        common_chat_msg_content_part msg_part;
-                        msg_part.type = type;
-                        if (type == "text" || type == "media_marker") {
-                            msg_part.text = part.at("text");
-                        } else if (type == "tool_reference") {
-                            // names a request tool, the template renders its definition
-                            msg_part.name = part.at("name");
-                        } else {
+                        if (type != "text" && type != "media_marker") {
                             throw std::invalid_argument("Unsupported content part type: " + type.dump());
                         }
+                        common_chat_msg_content_part msg_part;
+                        msg_part.type = type;
+                        msg_part.text = part.at("text");
                         msg.content_parts.push_back(msg_part);
                     }
                 } else if (!content.is_null()) {
@@ -440,41 +428,6 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
             if (message.contains("tool_call_id")) {
                 msg.tool_call_id = message.at("tool_call_id");
             }
-            {
-                json extra = json::object();
-                for (const auto & [k, v] : message.items()) {
-                    if (k != "role" && k != "content" && k != "tool_calls" && k != "reasoning_content" &&
-                        k != "name" && k != "tool_call_id") {
-                        extra[k] = v;
-                    }
-                }
-                if (has_tool_calls) {
-                    // per tool call: keys besides type/function/id, and function keys besides name/arguments
-                    json tc_extra = json::array();
-                    bool any      = false;
-                    for (const auto & tool_call : message.at("tool_calls")) {
-                        json e = json::object();
-                        for (const auto & [k, v] : tool_call.items()) {
-                            if (k != "type" && k != "function" && k != "id") {
-                                e[k] = v;
-                            }
-                        }
-                        for (const auto & [k, v] : tool_call.at("function").items()) {
-                            if (k != "name" && k != "arguments") {
-                                e["function"][k] = v;
-                            }
-                        }
-                        any = any || !e.empty();
-                        tc_extra.push_back(e);
-                    }
-                    if (any) {
-                        extra["__tool_call_extras"] = tc_extra;
-                    }
-                }
-                if (!extra.empty()) {
-                    msg.extra_json = extra.dump();
-                }
-            }
 
             msgs.push_back(msg);
         }
@@ -563,50 +516,6 @@ static json render_message_to_json(const std::vector<common_chat_msg> & msgs, co
     return messages_inp_normalizer(c).normalize(messages);
 }
 
-// DeepSeek V4.1 reads non-OpenAI message keys and joins content blocks with "\n\n" around images,
-// so keep media markers as separate text parts and restore the extra keys
-static json render_message_to_json_deepseek_v4_1(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {
-    json messages = json::array();
-    for (const auto & msg : msgs) {
-        json jmsg;
-        if (msg.contains_media() && msg.content.empty()) {
-            common_chat_msg typed = msg;
-            for (auto & part : typed.content_parts) {
-                part.type = "text";
-            }
-            jmsg = typed.to_json_oaicompat(/* concat_typed_text= */ false);
-        } else {
-            jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);
-        }
-        if (!msg.extra_json.empty()) {
-            const json extra = json::parse(msg.extra_json);
-            for (const auto & [k, v] : extra.items()) {
-                if (k == "__tool_call_extras") {
-                    if (!jmsg.contains("tool_calls")) {
-                        continue;
-                    }
-                    auto & tcs = jmsg.at("tool_calls");
-                    for (size_t i = 0; i < tcs.size() && i < v.size(); i++) {
-                        for (const auto & [tk, tv] : v[i].items()) {
-                            if (tk == "function") {
-                                for (const auto & [fk, fv] : tv.items()) {
-                                    tcs[i]["function"][fk] = fv;
-                                }
-                            } else {
-                                tcs[i][tk] = tv;
-                            }
-                        }
-                    }
-                } else if (!jmsg.contains(k)) {
-                    jmsg[k] = v;
-                }
-            }
-        }
-        messages.push_back(std::move(jmsg));
-    }
-    return messages_inp_normalizer(c).normalize(messages);
-}
-
 // DEPRECATED: only used in tests
 json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {
     jinja::caps c;
@@ -622,17 +531,13 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
 
     auto result = json::array();
     for (const auto & tool : tools) {
-        json function {
-            { "name",        tool.name },
-            { "description", tool.description },
-            { "parameters",  json::parse(tool.parameters) },
-        };
-        if (tool.defer_loading) {
-            function["defer_loading"] = true;
-        }
         result.push_back({
             { "type",     "function" },
-            { "function", function },
+            { "function", {
+                { "name", tool.name },
+                { "description", tool.description },
+                { "parameters", json::parse(tool.parameters) },
+            }},
         });
     }
     return result;
@@ -669,19 +574,10 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
                 }
 
                 const auto & function = tool.at("function");
-
-                // "defer_loading" sits on the function, or on the tool next to "type"; the function wins
-                const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
-                                                                        : tool.value("defer_loading", json(false));
-                if (!defer.is_boolean()) {
-                    throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
-                }
-
                 result.push_back({
                     /* .name = */ function.at("name"),
                     /* .description = */ function.value("description", ""),
                     /* .parameters = */ function.value("parameters", json::object()).dump(),
-                    /* .defer_loading = */ defer.get<bool>(),
                 });
             }
         }
@@ -804,70 +700,19 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
     return tmpls->has_explicit_template;
 }
 
-// renders a task template (e.g. "tts") with the given variables
-static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
-    jinja::context ctx(tmpl.source());
-    jinja::global_from_json(ctx, vars, false);
-    ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
-    ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
-    jinja::runtime runtime(ctx);
-    return jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
-}
-
-// LFM3 format detection: template renders deferred tools inside <additional_tools>...</additional_tools>
-static bool is_lfm3_template(const std::string & src) {
-    return src.find("<additional_tools>") != std::string::npos;
-}
-
-// LFM3-Audio: a known language code maps to its name, no language lets the model detect it
-static std::string lfm3_asr_system_prompt(const std::string & language) {
-    static const std::map<std::string, std::string> names = {
-        { "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
-        { "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
-        { "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
-        { "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
-    };
-    if (language.empty()) {
-        return "Perform ASR.";
-    }
-    std::string code = language;
-    std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); });
-    const auto it = names.find(code);
-    return "Perform ASR in " + (it != names.end() ? it->second : language) + ".";
-}
-
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {
     common_chat_prompt_preset asr_preset;
     asr_preset.system = "";
     asr_preset.user   = "Transcribe audio to text";
 
-    if (chat_templates && chat_templates->template_default) {
-        const std::string src = chat_templates->template_default->source();
-        if (is_lfm2_template(src)) {
-            asr_preset.system = "Perform ASR.";
-            asr_preset.user   = "";
-        } else if (is_lfm3_template(src)) {
-            asr_preset.system = lfm3_asr_system_prompt(language);
-            asr_preset.user   = "";
-            asr_preset.has_language = true;
-        }
+    if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {
+        asr_preset.system = "Perform ASR.";
+        asr_preset.user   = "";
     }
 
     return asr_preset;
 }
 
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language, const std::string & voice) {
-    if (!chat_templates || !chat_templates->template_tts) {
-        return "";
-    }
-    return common_chat_template_render(*chat_templates->template_tts, {
-        {"messages", json::array({{{"role", "user"}, {"content", text}}})},
-        {"language", language},
-        {"voice",    voice},
-    });
-}
-
 std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
     if (!variant.empty()) {
         if (variant == "tool_use") {
@@ -978,16 +823,6 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
             LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());
         }
     }
-    // task templates of the model, kept with a chat template override
-    if (model) {
-        if (const char * src = llama_model_chat_template(model, "tts")) {
-            try {
-                tmpls->template_tts = std::make_unique<common_chat_template>(src, token_bos, token_eos);
-            } catch (const std::exception & e) {
-                LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
-            }
-        }
-    }
     return tmpls;
 }
 
@@ -1373,13 +1208,11 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
     // DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
     // The template source contains the token as a variable assignment, not as a literal in markup.
     // V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
-    // V4.1 tag names have a leading space and its template does not name the dsml_token variable.
-    if ((src.find("dsml_token") != std::string::npos &&
-         src.find("DSML") != std::string::npos &&
-         (src.find("function_calls") != std::string::npos ||
-          src.find("tool_calls") != std::string::npos)) ||
-        src.find("｜DSML｜ calls>") != std::string::npos) {
-        LOG_DBG("Using specialized template: DeepSeek V3.2/V4/V4.1\n");
+    if (src.find("dsml_token") != std::string::npos &&
+        src.find("DSML") != std::string::npos &&
+        (src.find("function_calls") != std::string::npos ||
+         src.find("tool_calls") != std::string::npos)) {
+        LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
         return common_chat_params_init_deepseek_v3_2;
     }
 
@@ -1480,9 +1313,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
         workaround::trim_all_content(trimmed_messages);
         messages_to_render = &trimmed_messages;
     }
-    params.messages              = src.find("｜DSML｜ calls>") != std::string::npos ?
-                                       render_message_to_json_deepseek_v4_1(*messages_to_render, tmpl.original_caps()) :
-                                       render_message_to_json(*messages_to_render, tmpl.original_caps());
+    params.messages              = render_message_to_json(*messages_to_render, tmpl.original_caps());
     params.tool_choice           = inputs.tool_choice;
     params.reasoning_format      = inputs.reasoning_format;
     params.enable_thinking       = inputs.enable_thinking;
@@ -1803,9 +1634,6 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena &          src_pars
     //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
 
     common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
-    if (!is_partial) {
-        flags |= COMMON_PEG_PARSE_FLAG_COMPLETE;
-    }
     if (params.debug) {
         flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
     }
@@ -1943,13 +1771,12 @@ const common_chat_msg & common_chat_session::finish(const common_chat_input & ch
     return result;
 }
 
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
     GGML_ASSERT(chat_templates != nullptr);
     GGML_ASSERT(chat_templates->template_default != nullptr);
-    // take the more expressive template when available
-    const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps
-                                                                      : chat_templates->template_default->caps;
-    common_json result = caps.to_map();
-    result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
-    return result;
+    if (chat_templates->template_tool_use != nullptr) {
+        // take the more expressive template when available
+        return chat_templates->template_tool_use->caps.to_map();
+    }
+    return chat_templates->template_default->caps.to_map();
 }

common/chat.h

@@ -45,7 +45,6 @@ struct common_chat_tool_call {
 struct common_chat_msg_content_part {
     std::string type;
     std::string text;
-    std::string name; // tool_reference: name of a tool from the request tools
 
     // TODO @ngxson : no known chat templates support reasoning_content in content parts yet
     //                this can be useful for models with interleaved thinking (like Kimi-K2)
@@ -53,7 +52,7 @@ struct common_chat_msg_content_part {
     // std::string reasoning_content;
 
     bool operator==(const common_chat_msg_content_part & other) const {
-        return type == other.type && text == other.text && name == other.name;
+        return type == other.type && text == other.text;
     }
 };
 
@@ -95,8 +94,6 @@ struct common_chat_msg {
     std::string                               reasoning_content;
     std::string                               tool_name;
     std::string                               tool_call_id;
-    // non-OpenAI message keys as a JSON object string, only for templates that read them (DeepSeek V4.1)
-    std::string                               extra_json;
 
     common_json to_json_oaicompat(bool concat_typed_text = false) const;
 
@@ -227,9 +224,6 @@ struct common_chat_tool {
     std::string name;
     std::string description;
     std::string parameters;
-
-    // withheld from the system prompt, a tool_reference reveals it later in the conversation
-    bool defer_loading = false;
 };
 
 enum common_chat_tool_choice {
@@ -472,7 +466,7 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_to
 common_json common_chat_tool_parameters(const common_json & function);
 
 // get template caps, useful for reporting to server /props endpoint
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
 
 common_chat_prompt common_chat_template_direct_apply(
     const common_chat_template & tmpl,
@@ -491,12 +485,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
 struct common_chat_prompt_preset {
     std::string system;
     std::string user;
-    bool has_language = false; // the preset already names the requested language
 };
 
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
-
-// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
-// language and voice are optional, the template defines their defaults
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language = "", const std::string & voice = "");
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);

common/common.cpp

@@ -26,7 +26,6 @@
 #include <fstream>
 #include <iostream>
 #include <iterator>
-#include <random>
 #include <regex>
 #include <sstream>
 #include <string>
@@ -219,23 +218,6 @@ int32_t common_cpu_get_num_math() {
             }
         }
     }
-#elif defined(__riscv) && defined(__linux__)
-    std::ifstream cpuinfo("/proc/cpuinfo");
-    std::string line;
-    int32_t n_a100 = 0;
-    while (std::getline(cpuinfo, line)) {
-        std::istringstream fields(line);
-        std::string key;
-        std::string separator;
-        uint64_t marchid;
-        if (fields >> key >> separator >> std::hex >> marchid &&
-            key == "marchid" && separator == ":" && marchid == UINT64_C(0x8000000041000002)) {
-            ++n_a100;
-        }
-    }
-    if (n_a100 > 0) {
-        return n_a100;
-    }
 #elif defined(__powerpc64__) || defined(__powerpc__)
     int32_t smt_factor = 1;
     int phy_cpus = common_cpu_get_num_physical_cores();
@@ -759,56 +741,50 @@ static inline bool glob_class_match(const char c, const char * pattern, const ch
 }
 
 // simple glob: * matches non-/ chars, ** matches anything including /, [] matches character class
-// only wildcards recurse, too many wildcards: no match
-static bool glob_match(const char * pattern, const char * str, int depth = 0) {
-    if (depth >= COMMON_RECURSIVE_LIMIT) {
+static inline bool glob_match(const char * pattern, const char * str) {
+    if (*pattern == '\0') {
+        return *str == '\0';
+    }
+    if (pattern[0] == '*' && pattern[1] == '*') {
+        const char * p = pattern + 2;
+        if (glob_match(p, str)) return true;
+        if (*str != '\0') return glob_match(pattern, str + 1);
         return false;
     }
-    while (*pattern != '\0') {
-        if (pattern[0] == '*' && pattern[1] == '*') {
-            const char * p = pattern + 2;
-            for (;; str++) {
-                if (glob_match(p, str, depth + 1)) return true;
-                if (*str == '\0') return false;
-            }
+    if (*pattern == '*') {
+        const char * p = pattern + 1;
+        for (; *str != '\0' && *str != '/'; str++) {
+            if (glob_match(p, str)) return true;
         }
-        if (*pattern == '*') {
-            const char * p = pattern + 1;
-            for (; *str != '\0' && *str != '/'; str++) {
-                if (glob_match(p, str, depth + 1)) return true;
-            }
-            return glob_match(p, str, depth + 1);
-        }
-        if (*pattern == '?' && *str != '\0' && *str != '/') {
-            pattern++;
-            str++;
-            continue;
-        }
-        if (*pattern == '[') {
-            const char * class_end = pattern + 1;
-            // If first character after '[' is ']' or '-', treat it as literal
-            if (*class_end == ']' || *class_end == '-') {
-                class_end++;
-            }
-            while (*class_end != '\0' && *class_end != ']') {
-                class_end++;
-            }
-            if (*class_end == ']') {
-                if (*str == '\0' || !glob_class_match(*str, pattern + 1, class_end)) return false;
-                pattern = class_end + 1;
-                str++;
-                continue;
-            }
-            if (*str != '[') return false;
-            pattern++;
-            str++;
-            continue;
-        }
-        if (*pattern != *str) return false;
-        pattern++;
-        str++;
+        return glob_match(p, str);
     }
-    return *str == '\0';
+    if (*pattern == '?' && *str != '\0' && *str != '/') {
+        return glob_match(pattern + 1, str + 1);
+    }
+    if (*pattern == '[') {
+        const char * class_end = pattern + 1;
+        // If first character after '[' is ']' or '-', treat it as literal
+        if (*class_end == ']' || *class_end == '-') {
+            class_end++;
+        }
+        while (*class_end != '\0' && *class_end != ']') {
+            class_end++;
+        }
+        if (*class_end == ']') {
+            if (*str == '\0') return false;
+            bool matched = glob_class_match(*str, pattern + 1, class_end);
+            return matched && glob_match(class_end + 1, str + 1);
+        } else {
+            if (*str == '[') {
+                return glob_match(pattern + 1, str + 1);
+            }
+            return false;
+        }
+    }
+    if (*pattern == *str) {
+        return glob_match(pattern + 1, str + 1);
+    }
+    return false;
 }
 
 bool glob_match(const std::string & pattern, const std::string & str) {
@@ -895,7 +871,27 @@ bool fs_validate_filename(const std::string & filename, bool allow_subdirs) {
     return true;
 }
 
+#include <iostream>
+
+
 #ifdef _WIN32
+std::wstring utf8_to_wstring(const std::string & str) {
+    if (str.empty()) {
+        return std::wstring();
+    }
+
+    int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);
+
+    if (size <= 0) {
+        return std::wstring();
+    }
+
+    std::wstring wstr(size, 0);
+    MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);
+
+    return wstr;
+}
+
 std::string wstring_to_utf8(const std::wstring & str) {
     if (str.empty()) {
         return std::string();
@@ -914,14 +910,16 @@ std::string wstring_to_utf8(const std::wstring & str) {
 }
 #endif
 
-void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
-    // a random suffix gives every writer its own temporary file, across processes and containers
-    std::random_device rd;
-    const uint64_t id = ((uint64_t) rd() << 32) | rd();
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path) {
+    const auto value = path.u8string();
+    return std::string(value.begin(), value.end());
+}
 
+void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
     std::error_code ec;
     std::filesystem::path path_tmp = path;
-    path_tmp += "." + std::to_string(id) + ".tmp";
+    path_tmp += ".tmp";
 
     if (path.has_parent_path()) {
         std::filesystem::create_directories(path.parent_path(), ec);
@@ -937,7 +935,7 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat
 
     if (file.fail() || ec) {
         std::filesystem::remove(path_tmp, ec);
-        throw std::runtime_error("failed to write file: " + path.string());
+        throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
     }
 }
 
@@ -963,9 +961,20 @@ void common_set_env(const std::string & name, const std::string & value) {
 #endif
 }
 
+std::filesystem::path common_get_path_from_env(const std::string & name) {
+#if defined(_WIN32)
+    const std::wstring wname = utf8_to_wstring(name);
+    const wchar_t * wvalue = _wgetenv(wname.c_str());
+    return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
+#else
+    const char * value = std::getenv(name.c_str());
+    return value ? std::filesystem::path(value) : std::filesystem::path();
+#endif
+}
+
 #if !defined(_WIN32)
 static std::filesystem::path get_home_directory() {
-    std::filesystem::path home = common_get_env("HOME");
+    std::filesystem::path home = common_get_path_from_env("HOME");
     if (!home.empty()) {
         return home;
     }
@@ -978,19 +987,19 @@ static std::filesystem::path get_home_directory() {
 #endif
 
 std::filesystem::path fs_get_cache_directory() {
-    std::filesystem::path cache_directory = common_get_env("LLAMA_CACHE");
+    std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");
     if (!cache_directory.empty()) {
         return cache_directory;
     }
 #if defined(_WIN32)
-    cache_directory = common_get_env("LOCALAPPDATA");
+    cache_directory = common_get_path_from_env("LOCALAPPDATA");
     if (cache_directory.empty()) {
         throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
     }
 #elif defined(__APPLE__)
     cache_directory = get_home_directory() / "Library/Caches";
 #else
-    cache_directory = common_get_env("XDG_CACHE_HOME");
+    cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
     if (cache_directory.empty()) {
         cache_directory = get_home_directory() / ".cache";
     }
@@ -1001,12 +1010,12 @@ std::filesystem::path fs_get_cache_directory() {
 std::filesystem::path fs_get_config_directory() {
     std::filesystem::path config_directory;
 #if defined(_WIN32)
-    config_directory = common_get_env("APPDATA");
+    config_directory = common_get_path_from_env("APPDATA");
     if (config_directory.empty()) {
         throw std::runtime_error("Failed to find %APPDATA% directory");
     }
 #else
-    config_directory = common_get_env("XDG_CONFIG_HOME");
+    config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
     if (config_directory.empty()) {
         config_directory = get_home_directory() / ".config";
     }
@@ -1020,9 +1029,9 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) {
     std::error_code ec;
     common_create_directories(cache_directory, ec);
     if (ec) {
-        throw std::runtime_error("failed to create cache directory: " + cache_directory.string());
+        throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
     }
-    return cache_directory / filename;
+    return cache_directory / std::filesystem::u8path(filename);
 }
 
 //
@@ -1167,9 +1176,6 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_LFM2_D1_OMNI,   "lfm2-d1-omni"  },
     { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     { COMMON_DECISION_TYPE_H2O_LIGHTNING,  "h2o-lightning" },
-    { COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
-    { COMMON_DECISION_TYPE_LFM3_D1,        "lfm3-d1"       },
-    { COMMON_DECISION_TYPE_LFM3_D1_OMNI,   "lfm3-d1-omni"  },
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 
@@ -1235,50 +1241,44 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {
     return info;
 }
 
-void common_fit_apply(common_params & params, llama_model_params & mparams, llama_context_params & cparams) {
-    if (!params.fit_params) {
-        return;
-    }
-
-    COM_TRC("%s", "fitting params to device memory ...\n");
-    COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
-
-    // the draft context is created from the same base params and follows the main context, fit both together
-    const bool has_draft = params.speculative.has_dft();
-    const bool spec_mtp  = std::find(params.speculative.types.begin(), params.speculative.types.end(),
-        COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
-
-    common_params params_dft = common_base_params_to_speculative(params);
-
-    auto mparams_dft = common_model_params_to_llama(params_dft);
-    auto cparams_dft = common_context_params_to_llama(params_dft);
-    if (spec_mtp) {
-        cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
-    }
-    cparams_dft.n_rs_seq = 0;
-
-    const common_fit_extra_model extra = {
-        /*.path_model   =*/ params_dft.model.path.c_str(),
-        /*.mparams      =*/ &mparams_dft,
-        /*.cparams      =*/ &cparams_dft,
-        /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
-    };
-
-    common_fit_params(params.model.path.c_str(), &mparams, &cparams,
-        params.tensor_split,
-        params.tensor_buft_overrides.data(),
-        params.fit_params_target.data(),
-        params.fit_params_min_ctx,
-        has_draft || spec_mtp ? &extra : nullptr,
-        params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
-}
-
 common_init_result::common_init_result(common_params & params, bool model_only) :
     pimpl(new impl{}) {
     auto mparams = common_model_params_to_llama(params);
     auto cparams = common_context_params_to_llama(params);
 
-    common_fit_apply(params, mparams, cparams);
+    if (params.fit_params) {
+        COM_TRC("%s", "fitting params to device memory ...\n");
+        COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
+
+        // the draft context is created from the same base params and follows the main context, fit both together
+        const bool has_draft = params.speculative.has_dft();
+        const bool spec_mtp  = std::find(params.speculative.types.begin(), params.speculative.types.end(),
+            COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
+
+        common_params params_dft = common_base_params_to_speculative(params);
+
+        auto mparams_dft = common_model_params_to_llama(params_dft);
+        auto cparams_dft = common_context_params_to_llama(params_dft);
+        if (spec_mtp) {
+            cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
+        }
+        cparams_dft.n_rs_seq = 0;
+
+        const common_fit_extra_model extra = {
+            /*.path_model   =*/ params_dft.model.path.c_str(),
+            /*.mparams      =*/ &mparams_dft,
+            /*.cparams      =*/ &cparams_dft,
+            /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
+        };
+
+        common_fit_params(params.model.path.c_str(), &mparams, &cparams,
+            params.tensor_split,
+            params.tensor_buft_overrides.data(),
+            params.fit_params_target.data(),
+            params.fit_params_min_ctx,
+            has_draft || spec_mtp ? &extra : nullptr,
+            params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
+    }
 
     llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
     if (model == NULL) {
@@ -1296,29 +1296,19 @@ common_init_result::common_init_result(common_params & params, bool model_only)
     // these decision models return a score for each token via the embeddings output
     // TODO: maybe improve this in the future
     const auto decision_type = common_get_decision_type(model);
-    // minicpm-ocr-jev reads its class scores from the pooled output, the pooling type is the one of the GGUF
     if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
-        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2 ||
-        decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
-        const auto pooling_type = decision_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV ? LLAMA_POOLING_TYPE_UNSPECIFIED : LLAMA_POOLING_TYPE_NONE;
-
+        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
         params.embedding    = true;
-        params.pooling_type = pooling_type;
+        params.pooling_type = LLAMA_POOLING_TYPE_NONE;
 
         cparams.embeddings            = true;
-        cparams.pooling_type          = pooling_type;
+        cparams.pooling_type          = LLAMA_POOLING_TYPE_NONE;
         cparams.n_outputs_max         = cparams.n_batch;
         cparams.n_outputs_max_per_seq = 1;
 
         LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
     }
 
-    if (llama_model_classifier_free_guidance(model)) {
-        params.kv_unified   = true;
-        cparams.kv_unified  = true;
-        cparams.n_seq_max  *= 2;
-    }
-
     // embeddings need the whole batch in one ubatch, so n_batch must not be larger than n_ubatch
     // (server.cpp does this check for --embedding, but before the model is loaded)
     if (cparams.embeddings && cparams.n_batch > cparams.n_ubatch) {
@@ -1459,7 +1449,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
         if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
         if (params.control_vector_layer_end   <= 0) params.control_vector_layer_end   = llama_model_n_layer(model);
 
-        const auto cvec = common_control_vector_load(params.control_vectors, llama_model_n_layer(model));
+        const auto cvec = common_control_vector_load(params.control_vectors);
         if (cvec.n_embd == -1) {
             return res;
         }
@@ -1981,7 +1971,7 @@ float common_embd_similarity_cos(const float * embd1, const float * embd2, int n
 // Control vector utils
 //
 
-static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info, int32_t n_layer) {
+static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info) {
     common_control_vector_data result = { -1, {} };
 
     ggml_context * ctx = nullptr;
@@ -2022,10 +2012,6 @@ static common_control_vector_data common_control_vector_load_one(const common_co
             COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str());
             result.n_embd = -1;
             break;
-        } else if (layer_idx > n_layer) {
-            COM_ERR("direction tensor layer index %d exceeds the model layer count %d in %s\n", layer_idx, n_layer, load_info.fname.c_str());
-            result.n_embd = -1;
-            break;
         }
 
         struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());
@@ -2049,10 +2035,10 @@ static common_control_vector_data common_control_vector_load_one(const common_co
         }
 
         // extend if necessary - do not store data for layer 0 (it's not used)
-        result.data.resize(std::max(result.data.size(), (size_t) result.n_embd * layer_idx), 0.0f);
+        result.data.resize(std::max(result.data.size(), static_cast<size_t>(result.n_embd * layer_idx)), 0.0f);
 
         const float * src = (const float *) tensor->data;
-        float * dst = result.data.data() + (size_t) result.n_embd * (layer_idx - 1);  // layer 1 at [0]
+        float * dst = result.data.data() + result.n_embd * (layer_idx - 1);  // layer 1 at [0]
         for (int j = 0; j < result.n_embd; j++) {
             dst[j] += src[j] * load_info.strength;  // allows multiple directions for same layer in same file
         }
@@ -2070,11 +2056,11 @@ static common_control_vector_data common_control_vector_load_one(const common_co
     return result;
 }
 
-common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer) {
+common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos) {
     common_control_vector_data result = { -1, {} };
 
     for (const auto & info : load_infos) {
-        auto cur = common_control_vector_load_one(info, n_layer);
+        auto cur = common_control_vector_load_one(info);
 
         if (cur.n_embd == -1) {
             result.n_embd = -1;
@@ -2250,16 +2236,6 @@ int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq
     return size() - 1;
 }
 
-int32_t common_batch::add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output) {
-    token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} };
-    t.embd_row = row;
-    for (int32_t j = 0; j < n_pos; ++j) {
-        t.pos[j] = pos[j];
-    }
-    tokens.push_back(t);
-    return size() - 1;
-}
-
 llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
     GGML_ASSERT(batch && "common_batch was not initialized with a context");
     GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size());
@@ -2280,12 +2256,6 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
             if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
                 GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
             }
-        } else if (t.embd_row >= 0) {
-            idx = llama_batch_ext_add_embd_row(res, t.seq_id, t.embd_row);
-            if (idx < 0) {
-                GGML_ABORT("%s: failed to add embd row %d at index %d (error %d, n = %d)\n", __func__, t.embd_row, i, idx, n);
-            }
-            llama_batch_ext_set_pos(res, idx, t.pos.data());
         } else {
             idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
             if (idx < 0) {

common/common.h

@@ -31,8 +31,6 @@
 #define DIRECTORY_SEPARATOR '/'
 #endif // _WIN32
 
-#define COMMON_RECURSIVE_LIMIT 128
-
 #define COM_DBG(fmt, ...) LOG_DBG("cmn  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
 #define COM_TRC(fmt, ...) LOG_TRC("cmn  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
 #define COM_INF(fmt, ...) LOG_INF("cmn  %12.*s: " fmt, 12, __func__, __VA_ARGS__)
@@ -701,7 +699,6 @@ struct common_params {
     std::string models_dir    = "";     // directory containing models for the router server
     std::string models_preset = "";     // directory containing model presets for the router server
     int models_max = 4;                 // maximum number of models to load simultaneously
-    int models_memory_margin = 0;       // MiB of device memory to keep free when loading models (0 = no memory accounting)
     bool models_autoload = true;        // automatically load models when requested via the router server
     std::string models_preset_hf = "";  // show a warning about remote presets on router loaded (if not empty)
 
@@ -772,7 +769,6 @@ struct common_params {
 
     // TTS params
     std::string tts_lang = "";
-    std::string tts_voice = "";
     std::string tts_speaker_file = "";
 
     bool is_gen_docs = false; // whether we are running inside llama-gen-docs
@@ -906,9 +902,13 @@ bool glob_match(const std::string & pattern, const std::string & str);
 //
 
 #ifdef _WIN32
-std::string wstring_to_utf8(const std::wstring & str);
+std::wstring utf8_to_wstring(const std::string & str);
+std::string  wstring_to_utf8(const std::wstring & str);
 #endif
 
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path);
+
 //
 // Environment utils
 //
@@ -918,6 +918,9 @@ std::string wstring_to_utf8(const std::wstring & str);
 std::string common_get_env(const std::string & name);
 void        common_set_env(const std::string & name, const std::string & value);
 
+// reads a path from the environment, an unset variable gives an empty path
+std::filesystem::path common_get_path_from_env(const std::string & name);
+
 //
 // Filesystem utils
 //
@@ -973,9 +976,6 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
     COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
-    COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
-    COMMON_DECISION_TYPE_LFM3_D1,       // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
-    COMMON_DECISION_TYPE_LFM3_D1_OMNI,  // same as lfm3-d1
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 
@@ -1015,9 +1015,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
 struct llama_model_params   common_model_params_to_llama  (      common_params & params);
 struct llama_context_params common_context_params_to_llama(const common_params & params);
 
-// apply --fit to mparams and cparams, no-op if params.fit_params is false
-void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);
-
 // clear LoRA adapters from context, then apply new list of adapters
 void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);
 
@@ -1092,7 +1089,6 @@ struct common_batch {
         llama_embd   state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
         std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
         int32_t      decision_order = 0; // see llama_batch_ext_set_decision_order()
-        int32_t      embd_row = -1; // see add_embd_row(), -1 if none
     };
 
     std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
@@ -1135,10 +1131,6 @@ struct common_batch {
     // pos points to n_pos positions
     int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
 
-    // add an entry whose embedding is row `row` of the layer input device buffer of the context's ctx_other
-    // pos points to n_pos positions
-    int32_t add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output);
-
     int32_t size() const { return (int32_t) tokens.size(); }
 };
 
@@ -1235,9 +1227,8 @@ struct common_control_vector_load_info {
 };
 
 // Load control vectors, scale each by strength, and add them together.
-// Direction layer indices must be in [1, n_layer].
 // On error, returns {-1, empty}
-common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer);
+common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos);
 
 //
 // Split utils
@@ -1289,20 +1280,6 @@ enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);
 // prompt utils
 //
 
-// leave default-inserted elements uninitialized; callers must fill them before use
-template <typename T>
-struct common_no_init_allocator : std::allocator<T> {
-    template <typename U> struct rebind { using other = common_no_init_allocator<U>; };
-
-    common_no_init_allocator() = default;
-    template <typename U> common_no_init_allocator(const common_no_init_allocator<U> &) {}
-
-    template <typename U> void construct(U * p) { ::new ((void *) p) U; }
-};
-
-// sequence state data, filled by llama_state_seq_get_data_ext()
-using common_state_data = std::vector<uint8_t, common_no_init_allocator<uint8_t>>;
-
 struct common_prompt_checkpoint {
     int64_t n_tokens;
 
@@ -1312,8 +1289,8 @@ struct common_prompt_checkpoint {
     llama_pos pos_min;
     llama_pos pos_max;
 
-    common_state_data data_tgt;
-    common_state_data data_dft;
+    std::vector<uint8_t> data_tgt;
+    std::vector<uint8_t> data_dft;
 
     // (optional) speculative-decoding implementation state stashed with the checkpoint
     // (e.g. eagle3's deferred-boundary g_embd row)

common/download.cpp

@@ -4,7 +4,6 @@
 #include "common.h"
 #include "log.h"
 #include "download.h"
-#include "file-lock.h"
 #include "hf-cache.h"
 #include "json.h"
 
@@ -45,7 +44,7 @@ static const std::string PARTIAL_SUFFIX = ".downloadInProgress";
 
 static void write_etag(const std::string & path, const std::string & etag) {
     const std::string etag_path = path + ".etag";
-    fs_write_atomic(etag_path, etag);
+    fs_write_atomic(std::filesystem::u8path(etag_path), etag);
     LOG_DBG("%s: file etag saved: %s\n", __func__, etag_path.c_str());
 }
 
@@ -78,18 +77,6 @@ std::pair<std::string, std::string> common_download_split_repo_tag(const std::st
     return {hf_repo, tag};
 }
 
-// file name from url, without the query string
-static std::string url_get_filename(const std::string & url) {
-    std::string filename = url;
-    if (auto pos = filename.rfind('/'); pos != std::string::npos) {
-        filename = filename.substr(pos + 1);
-    }
-    if (auto pos = filename.find('?'); pos != std::string::npos) {
-        filename = filename.substr(0, pos);
-    }
-    return filename;
-}
-
 class ProgressBar : public common_download_callback {
     static inline std::mutex mutex;
     static inline std::map<const ProgressBar *, int> lines;
@@ -113,8 +100,14 @@ public:
     ProgressBar() = default;
 
     void on_start(const common_download_progress & p) override {
-        filename = url_get_filename(p.url);
+        filename = p.url;
 
+        if (auto pos = filename.rfind('/'); pos != std::string::npos) {
+            filename = filename.substr(pos + 1);
+        }
+        if (auto pos = filename.find('?'); pos != std::string::npos) {
+            filename = filename.substr(0, pos);
+        }
         for (size_t i = 0; i < filename.size(); ++i) {
             if ((filename[i] & 0xC0) != 0x80) {
                 if (len++ == 39) {
@@ -264,61 +257,15 @@ static bool common_pull_file(httplib::Client & cli,
 static int common_download_file_single_online(const std::string & url,
                                               const std::string & path,
                                               const common_download_opts & opts,
-                                              bool skip_etag,
-                                              const std::string & lock_path,
-                                              const std::string & alt_path) {
+                                              bool skip_etag) {
     static const int max_attempts        = 3;
     static const int retry_delay_seconds = 2;
 
-    const std::string path_temporary = path + PARTIAL_SUFFIX;
-    const std::string path_lock = lock_path.empty() ? path + ".lock" : lock_path;
-
-    common_download_progress p;
-    p.url = url;
-
-    if (opts.callback) {
-        opts.callback->on_start(p);
-    }
-
-    // the completion callback may itself take the same lock (the HF cache
-    // finalizes blobs under it), so every exit goes through finish()
-    common_file_lock lock(path_lock);
-    auto finish = [&](int status) {
-        lock.close();
-        if (opts.callback) {
-            opts.callback->on_done(p, is_http_status_ok(status));
-        }
-        return status;
-    };
-
-    // alt_path is the HF snapshot: the blob may have been moved there when symlinks are unavailable
-    auto cache_hit = [&] {
-        return std::filesystem::exists(path) || (!alt_path.empty() && std::filesystem::exists(alt_path));
-    };
-
-    if (skip_etag && cache_hit()) {
-        LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
-        return finish(304); // 304 Not Modified - fake cached response
-    }
-
-    // take the lock before touching the cache, so two processes cannot race on the same file
-    bool waited = false;
-    if (!lock.acquire([&] {
-            if (!waited) {
-                LOG_INF("%s: file '%s' is being downloaded by another process, waiting...\n", __func__, url_get_filename(url).c_str());
-                waited = true;
-            }
-            return !(opts.callback && opts.callback->is_cancelled());
-        })) {
-        return finish(-1);
-    }
-
-    // under the lock: another process may have finished while we waited
-    bool file_exists = cache_hit();
+    const bool file_exists = std::filesystem::exists(std::filesystem::u8path(path));
 
     if (file_exists && skip_etag) {
         LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str());
-        return finish(304); // 304 Not Modified - fake cached response
+        return 304; // 304 Not Modified - fake cached response
     }
 
     auto [cli, parts] = common_http_client(url);
@@ -347,9 +294,9 @@ static int common_download_file_single_online(const std::string & url,
         LOG_TRC("%s: HEAD failed, status: %d\n", __func__, head ? head->status : -1);
         if (file_exists) {
             LOG_TRC("%s: using cached file (HEAD failed): %s\n", __func__, path.c_str());
-            return finish(304); // 304 Not Modified - fake cached response
+            return 304; // 304 Not Modified - fake cached response
         }
-        return finish(head ? head->status : -1);
+        return head ? head->status : -1;
     }
 
     std::string etag;
@@ -357,6 +304,8 @@ static int common_download_file_single_online(const std::string & url,
         etag = head->get_header_value("ETag");
     }
 
+    common_download_progress p;
+    p.url = url;
     if (head->has_header("Content-Length")) {
         try {
             p.total = std::stoull(head->get_header_value("Content-Length"));
@@ -373,16 +322,16 @@ static int common_download_file_single_online(const std::string & url,
     if (file_exists) {
         if (etag.empty()) {
             LOG_DBG("%s: using cached file (no server etag): %s\n", __func__, path.c_str());
-            return finish(304); // 304 Not Modified - fake cached response
+            return 304; // 304 Not Modified - fake cached response
         }
         if (!last_etag.empty() && last_etag == etag) {
             LOG_DBG("%s: using cached file (same etag): %s\n", __func__, path.c_str());
-            return finish(304); // 304 Not Modified - fake cached response
+            return 304; // 304 Not Modified - fake cached response
         }
         // pass this point, the file exists but is different from the server version, so we need to redownload it
         if (remove(path.c_str()) != 0) {
             LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str());
-            return finish(-1);
+            return -1;
         }
     }
 
@@ -392,6 +341,7 @@ static int common_download_file_single_online(const std::string & url,
     }
 
     bool success = false;
+    const std::string path_temporary = path + PARTIAL_SUFFIX;
     int delay = retry_delay_seconds;
 
     if (!skip_etag && !etag.empty()) {
@@ -399,7 +349,7 @@ static int common_download_file_single_online(const std::string & url,
             LOG_WRN("%s: partial file does not match the server etag, discarding it: %s\n", __func__, path_temporary.c_str());
             if (remove(path_temporary.c_str()) != 0) {
                 LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str());
-                return finish(-1);
+                return -1;
             }
         }
         write_etag(path_temporary, etag);
@@ -409,7 +359,7 @@ static int common_download_file_single_online(const std::string & url,
         // report the resume offset, so a resumed file does not start at 0
         p.downloaded = supports_ranges && std::filesystem::exists(path_temporary) ?
                        std::filesystem::file_size(path_temporary) : 0;
-        opts.callback->on_update(p);
+        opts.callback->on_start(p);
     }
 
     for (int i = 0; i < max_attempts; ++i) {
@@ -453,12 +403,16 @@ static int common_download_file_single_online(const std::string & url,
         }
     }
 
+    if (opts.callback) {
+        opts.callback->on_done(p, success);
+    }
     // a stopped download keeps path_temporary, so the next attempt resumes from it
     if (!success) {
         LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
+        return -1; // max attempts reached
     }
 
-    return finish(success ? head->status : -1);
+    return head->status;
 }
 
 std::pair<long, std::vector<char>> common_remote_get_content(const std::string          & url,
@@ -498,19 +452,17 @@ std::pair<long, std::vector<char>> common_remote_get_content(const std::string
 int common_download_file_single(const std::string & url,
                                 const std::string & path,
                                 const common_download_opts & opts,
-                                bool skip_etag,
-                                const std::string & lock_path,
-                                const std::string & alt_path) {
+                                bool skip_etag) {
     if (!opts.offline) {
         ProgressBar tty_cb;
         common_download_opts online_opts = opts;
         if (!online_opts.callback) {
             online_opts.callback = &tty_cb;
         }
-        return common_download_file_single_online(url, path, online_opts, skip_etag, lock_path, alt_path);
+        return common_download_file_single_online(url, path, online_opts, skip_etag);
     }
 
-    if (!std::filesystem::exists(path)) {
+    if (!std::filesystem::exists(std::filesystem::u8path(path))) {
         LOG_ERR("%s: required file is not available in cache (offline mode): %s\n", __func__, path.c_str());
         return -1;
     }
@@ -747,8 +699,7 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
     }
 
     for (const auto & t : tags) {
-        // match the tag as a full token, so ":F16" does not match a BF16 file
-        std::regex pattern("(^|[-./])" + t + "[.-]", std::regex::icase);
+        std::regex pattern(t + "[.-]", std::regex::icase);
         for (const auto & f : files) {
             if (gguf_filename_is_model(f.path) &&
                 std::regex_search(f.path, pattern)) {
@@ -870,7 +821,7 @@ void common_download_run_tasks(const std::vector<common_download_task> & tasks)
     for (const auto & task : tasks) {
         futures.push_back(std::async(std::launch::async,
             [&task]() {
-                return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf, task.lock_path, task.alt_path);
+                return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf);
             }
         ));
     }
@@ -1005,7 +956,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
         std::string model_filename = repo;
         std::replace(model_filename.begin(), model_filename.end(), '/', '_');
         model_filename += "_" + tag + ".gguf";
-        std::string local_path = fs_get_cache_file(model_filename).string();
+        std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
 
         const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
         common_download_opts opts;
@@ -1211,7 +1162,7 @@ bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const
     for (const auto & s : sidecars) {
         const auto & f = plan.*s.file;
         if (!f.local_path.empty()) {
-            to_remove.emplace_back(f.local_path);
+            to_remove.emplace_back(std::filesystem::u8path(f.local_path));
         }
     }
     if (to_remove.empty()) {

common/download.h

@@ -69,15 +69,12 @@ struct common_download_task {
     std::string local_path;
     std::function<void()> on_done;
     bool is_hf = false;
-    std::string lock_path;
-    std::string alt_path; // HF snapshot path, also treated as cached (the blob may be moved there)
 
     common_download_task() = default;
     common_download_task(hf_cache::hf_file f,
             const common_download_opts & opts,
             std::function<void()> on_done = nullptr)
-        : opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_hf(true),
-          lock_path(hf_cache::get_lock_path(f.repo_id, f.oid)), alt_path(f.final_path) {}
+        : opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_hf(true) {}
 };
 
 void common_download_run_tasks(const std::vector<common_download_task> & tasks);
@@ -95,14 +92,10 @@ std::string common_download_resolve_path(const std::string & hf_repo_with_tag, c
 // download single file from url to local path
 // returns status code or -1 on error
 // skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash)
-// lock_path: lock file used to serialize downloads of the same file; when empty, "<path>.lock" is used
-// alt_path: additional path treated as present in cache (see cache_hit in download.cpp)
 int common_download_file_single(const std::string & url,
                                 const std::string & path,
                                 const common_download_opts & opts = {},
-                                bool skip_etag = false,
-                                const std::string & lock_path = "",
-                                const std::string & alt_path = "");
+                                bool skip_etag = false);
 
 // resolve and download model from Docker registry
 // return local path to downloaded model file

common/file-lock.cpp

@@ -1,179 +0,0 @@
-#include "file-lock.h"
-
-#include "log.h"
-
-#include <chrono>
-#include <filesystem>
-#include <thread>
-
-#include <fcntl.h>
-#include <sys/stat.h>
-
-#if defined(_WIN32)
-#define WIN32_LEAN_AND_MEAN
-#ifndef NOMINMAX
-#define NOMINMAX
-#endif
-#include <windows.h>
-#include <io.h>
-#else
-#include <sys/file.h>
-#include <unistd.h>
-#include <cerrno>
-#include <cstring>
-#endif
-
-common_file_lock::common_file_lock(const std::string & lock_path) {
-    this->lock_path = lock_path;
-}
-
-#if defined(_WIN32)
-// the \\?\ prefix (for paths longer than 255 chars) is only accepted on absolute paths
-// with backslash separators: normalize first, then check the length of the absolute form
-static std::wstring as_extended_path(const std::string & path) {
-    // convert from the narrow encoding std::filesystem uses on this system
-    int wlen = MultiByteToWideChar(CP_ACP, 0, path.c_str(), (int) path.size(), nullptr, 0);
-    if (wlen <= 0) {
-        return std::wstring(path.begin(), path.end()); // best effort for exotic encodings
-    }
-    std::wstring wide(wlen, L'\0');
-    MultiByteToWideChar(CP_ACP, 0, path.c_str(), (int) path.size(), wide.data(), wlen);
-
-    if (wide.rfind(L"\\\\?\\", 0) == 0) {
-        return wide;
-    }
-
-    // GetFullPathNameW resolves relative paths and normalizes separators
-    DWORD size = GetFullPathNameW(wide.c_str(), 0, nullptr, nullptr);
-    if (size == 0) {
-        return wide;
-    }
-    std::wstring full(size, L'\0');
-    DWORD written = GetFullPathNameW(wide.c_str(), size, full.data(), nullptr);
-    if (written == 0 || written >= size) {
-        return wide;
-    }
-    full.resize(written);
-
-    if (full.size() <= 255) {
-        return wide; // short enough: open the path exactly as given
-    }
-    if (full.rfind(L"\\\\", 0) == 0) {
-        return L"\\\\?\\UNC\\" + full.substr(2); // \\server\share -> \\?\UNC\server\share
-    }
-    return L"\\\\?\\" + full;
-}
-#endif
-
-common_file_lock::status common_file_lock::try_lock() {
-    if (lock_path.empty()) {
-        return status::acquired; // no lock path, nothing to lock
-    }
-#if defined(_WIN32)
-    if (fd < 0) {
-        // opened with a wide path: extended paths cannot go through the narrow CRT
-        std::wstring lock_path_w = as_extended_path(lock_path);
-        fd = _wopen(lock_path_w.c_str(), _O_RDWR | _O_CREAT | _O_BINARY | _O_NOINHERIT, _S_IREAD | _S_IWRITE);
-        if (fd < 0) {
-            LOG_ERR("%s: cannot open lock file: %s\n", __func__, lock_path.c_str());
-            return status::error;
-        }
-    }
-    // lock the first byte, the file itself stays empty
-    OVERLAPPED ov = {};
-    locked = LockFileEx((HANDLE) _get_osfhandle(fd),
-                        LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &ov);
-    if (locked) {
-        return status::acquired;
-    }
-    if (GetLastError() == ERROR_LOCK_VIOLATION) {
-        return status::busy;
-    }
-    LOG_ERR("%s: file locking failed (%d)\n", __func__, (int) GetLastError());
-    return status::error;
-#else
-    if (fd < 0) {
-        int open_flags = O_RDWR | O_CREAT | O_CLOEXEC;
-#ifdef O_NOFOLLOW
-        open_flags |= O_NOFOLLOW;
-#endif
-        // group-writable, so users sharing a cache can wait on the lock
-        fd = open(lock_path.c_str(), open_flags, 0664);
-        if (fd < 0) {
-            LOG_ERR("%s: cannot open lock file: %s\n", __func__, lock_path.c_str());
-            return status::error;
-        }
-        // open() filters the mode through the umask
-        (void) fchmod(fd, 0664);
-    }
-
-    if (flock(fd, LOCK_EX | LOCK_NB) == 0) {
-        locked = true;
-        return status::acquired;
-    }
-    // EACCES is reported as contention on some filesystems
-    if (errno == EWOULDBLOCK || errno == EAGAIN || errno == EACCES) {
-        return status::busy;
-    }
-    // flock is not implemented (some NFS/FUSE mounts): report it instead of racing
-    LOG_ERR("%s: file locking failed (%s)\n", __func__, strerror(errno));
-    return status::error;
-#endif
-}
-
-bool common_file_lock::acquire(const std::function<bool()> & keep_waiting) {
-    // the lock file is opened by try_lock, its parent must exist by then
-    if (!lock_path.empty()) {
-        std::error_code ec;
-        auto lock_dir = std::filesystem::path(lock_path).parent_path();
-        if (!lock_dir.empty()) {
-            std::filesystem::create_directories(lock_dir, ec);
-            if (ec) {
-                LOG_ERR("%s: unable to create lock directory '%s': %s\n", __func__, lock_dir.string().c_str(), ec.message().c_str());
-                return false;
-            }
-        }
-    }
-
-    while (true) {
-        switch (try_lock()) {
-            case status::acquired:
-                return true;
-            case status::busy:
-                if (keep_waiting && !keep_waiting()) {
-                    return false;
-                }
-                std::this_thread::sleep_for(std::chrono::milliseconds(100));
-                continue;
-            case status::error:
-                return false;
-        }
-    }
-}
-
-// the lock file is never deleted: another process could then create a fresh
-// file at the same path and lock that
-void common_file_lock::close() {
-#if defined(_WIN32)
-    if (fd >= 0) {
-        if (locked) {
-            OVERLAPPED ov = {};
-            UnlockFileEx((HANDLE) _get_osfhandle(fd), 0, 1, 0, &ov);
-        }
-        _close(fd);
-    }
-#else
-    if (fd >= 0) {
-        if (locked) {
-            flock(fd, LOCK_UN);
-        }
-        ::close(fd);
-    }
-#endif
-    fd     = -1;
-    locked = false;
-}
-
-common_file_lock::~common_file_lock() {
-    close();
-}

common/file-lock.h

@@ -1,36 +0,0 @@
-#pragma once
-
-#include <functional>
-#include <string>
-
-// Cross-process exclusive lock (flock on POSIX, LockFileEx on Windows).
-// Lazy: acquire() opens the lock file, so a read-only cache works if the lock is never acquired.
-class common_file_lock {
-public:
-    common_file_lock(const std::string & lock_path);
-    ~common_file_lock();
-
-    // take the lock, waiting while another process holds it;
-    // keep_waiting is polled before each retry, false cancels the wait.
-    // Also returns false when locking is unavailable.
-    bool acquire(const std::function<bool()> & keep_waiting = {});
-
-    void close();
-
-    common_file_lock(const common_file_lock &) = delete;
-    common_file_lock & operator=(const common_file_lock &) = delete;
-
-private:
-    enum class status {
-        acquired,
-        busy,  // another process holds the lock
-        error, // file locking is unavailable: no exclusion is possible
-    };
-
-    status try_lock();
-
-    std::string lock_path;
-
-    int  fd     = -1;
-    bool locked = false;
-};

common/fit.cpp

@@ -1,8 +1,6 @@
 #include "fit.h"
 
-#include "common.h"
 #include "json.h"
-#include "llama-cpp.h"
 #include "log.h"
 
 #include "../src/llama-ext.h"
@@ -29,43 +27,31 @@ class common_params_fit_exception : public std::runtime_error {
     using std::runtime_error::runtime_error;
 };
 
-// RAII: send llama logs below min_level to the debug log
-struct common_fit_log_scope {
-    common_fit_log_scope(ggml_log_level min_level) : min_level(min_level) {
-        llama_log_get(&original_logger.callback, &original_logger.user_data);
-        llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
-            const common_fit_log_scope * self = (const common_fit_log_scope *) user_data;
-            const ggml_log_level level_eff = level >= self->min_level ? level : GGML_LOG_LEVEL_DEBUG;
-            self->original_logger.callback(level_eff, text, self->original_logger.user_data);
-        }, this);
-    }
-
-    ~common_fit_log_scope() {
-        llama_log_set(original_logger.callback, original_logger.user_data);
-    }
-
-    common_fit_log_scope(const common_fit_log_scope &) = delete;
-    common_fit_log_scope & operator=(const common_fit_log_scope &) = delete;
-
-private:
-    struct {
-        ggml_log_callback callback;
-        void * user_data;
-    } original_logger;
-    ggml_log_level min_level;
-};
-
 static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
         const char * path_model,
         const llama_model_params * mparams,
         const llama_context_params * cparams,
         std::vector<ggml_backend_dev_t> & devs,
         uint32_t & hp_ngl,
-        uint32_t & hp_ngl_skip,
         uint32_t & hp_n_ctx_train,
         uint32_t & hp_n_expert,
         ggml_log_level log_level) {
-    common_fit_log_scope log_scope(log_level);
+    struct user_data_t {
+        struct {
+            ggml_log_callback callback;
+            void * user_data;
+        } original_logger;
+        ggml_log_level min_level; // prints below this log level go to debug log
+    };
+    user_data_t ud;
+    llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);
+    ud.min_level = log_level;
+
+    llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
+        const user_data_t * ud = (const user_data_t *) user_data;
+        const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;
+        ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);
+    }, &ud);
 
     llama_model_params mparams_copy = *mparams;
     mparams_copy.no_alloc  = true;
@@ -73,12 +59,14 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
 
     llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
     if (model == nullptr) {
+        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
         throw std::runtime_error("failed to load model");
     }
 
     llama_context * ctx = llama_init_from_model(model, *cparams);
     if (ctx == nullptr) {
         llama_model_free(model);
+        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
         throw std::runtime_error("failed to create llama_context from model");
     }
 
@@ -153,8 +141,6 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
     if (mparams->load_mtp) {
         hp_ngl    += llama_model_n_layer_nextn(model);
     }
-    // without MTP the NextN layers hold no weights but the model loader still counts them for n_gpu_layers
-    hp_ngl_skip    = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model);
     hp_n_ctx_train = llama_model_n_ctx_train(model);
     hp_n_expert    = llama_model_n_expert(model);
 
@@ -162,93 +148,22 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
 
     llama_free(ctx);
     llama_model_free(model);
+    llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);
 
     return ret;
 }
 
-std::vector<common_memory_breakdown_row> common_fit_measure(
-        const char * path_model,
-        const llama_model_params * mparams,
-        const llama_context_params * cparams,
-        uint32_t * n_ctx_train,
-        uint32_t * n_ctx,
-        ggml_log_level log_level) {
-    common_fit_log_scope log_scope(log_level);
-
-    llama_model_params mparams_copy = *mparams;
-    mparams_copy.no_alloc  = true;
-    mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
-
-    llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy));
-    if (!model) {
-        throw std::runtime_error("failed to load model");
-    }
-
-    llama_context_ptr ctx(llama_init_from_model(model.get(), *cparams));
-    if (!ctx) {
-        throw std::runtime_error("failed to create llama_context from model");
-    }
-
-    if (n_ctx_train) {
-        *n_ctx_train = llama_model_n_ctx_train(model.get());
-    }
-    if (n_ctx) {
-        *n_ctx = llama_n_ctx(ctx.get());
-    }
-
-    return common_memory_breakdown_rows(ctx.get());
-}
-
-common_memory_reports common_measure_memory(common_params & params) {
-    const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
-    const char * path_model = params.model.path.c_str();
-
-    common_memory_reports res;
-
-    auto mparams = common_model_params_to_llama(params);
-    auto cparams = common_context_params_to_llama(params);
-
-    // max: what --fit picks now, no-op with -fit off
-    {
-        auto mparams_fit = mparams;
-        auto cparams_fit = cparams;
-        common_fit_apply(params, mparams_fit, cparams_fit);
-        res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
-    }
-
-    // min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
-    uint32_t n_ctx_min = cparams.n_ctx;
-    if (n_ctx_min == 0 && params.fit_params) {
-        const uint32_t n_streams = cparams.kv_unified ? 1 : std::max<uint32_t>(1, cparams.n_seq_max);
-        // -c 0 sets fit_params_min_ctx to UINT32_MAX, cap at n_ctx_train like --fit
-        n_ctx_min = (uint32_t) std::min<uint64_t>(
-            uint64_t(params.fit_params_min_ctx) * n_streams,
-            uint64_t(res.n_ctx_train) * n_streams);
-    }
-    if (n_ctx_min == 0 || n_ctx_min >= res.n_ctx_max) {
-        // nothing to reduce, or -fit off with the trained context
-        res.n_ctx_min = res.n_ctx_max;
-        res.info_min  = res.info_max;
-    } else {
-        cparams.n_ctx = n_ctx_min;
-        res.info_min  = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
-    }
-
-    return res;
-}
-
 common_device_memory_data_vec common_get_device_memory_data(
         const char * path_model,
         const llama_model_params * mparams,
         const llama_context_params * cparams,
         std::vector<ggml_backend_dev_t> & devs,
         uint32_t & hp_ngl,
-        uint32_t & hp_ngl_skip,
         uint32_t & hp_n_ctx_train,
         uint32_t & hp_n_expert,
         ggml_log_level log_level) {
     std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
-            path_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
+            path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
 
     common_device_memory_data_vec ret(impl.size());
     for (size_t i = 0; i < impl.size(); i++) {
@@ -274,7 +189,6 @@ static void common_params_fit_impl(
 
     std::vector<ggml_backend_dev_t> devs;
     uint32_t hp_ngl = 0; // hparams.n_gpu_layers
-    uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
     uint32_t hp_nct = 0; // hparams.n_ctx_train
     uint32_t hp_nex = 0; // hparams.n_expert
 
@@ -296,7 +210,6 @@ static void common_params_fit_impl(
         if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) {
             std::vector<ggml_backend_dev_t> devs_extra;
             uint32_t ngl_extra = 0;
-            uint32_t nsk_extra = 0;
             uint32_t nct_extra = 0;
             uint32_t nex_extra = 0;
 
@@ -308,7 +221,7 @@ static void common_params_fit_impl(
             dmds_t measured;
             try {
                 measured = common_get_device_memory_data_impl(
-                    extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
+                    extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
             } catch (const std::runtime_error & e) {
                 // the extra model is optional, fit the main model alone rather than giving up
                 LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
@@ -348,7 +261,7 @@ static void common_params_fit_impl(
     // step 1: get data for default parameters and check whether any changes are necessary in the first place
 
     LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
-    dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+    dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
 
     // saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
     const uint32_t n_ctx_max       = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct)    * n_streams, UINT32_MAX);
@@ -360,7 +273,7 @@ static void common_params_fit_impl(
         if (n_streams > 1) {
             LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
                 __func__, n_ctx_max, n_streams);
-            dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+            dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
         }
     }
     add_extra_memory(dmds_full);
@@ -499,7 +412,7 @@ static void common_params_fit_impl(
 
                     int64_t sum_projected_used_min_ctx = 0;
                     cparams->n_ctx = n_ctx_min_total;
-                    dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+                    dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
                     add_extra_memory(dmds_min_ctx);
                     if (nd == 0) {
                         sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
@@ -646,20 +559,6 @@ static void common_params_fit_impl(
         assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);
         uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides
 
-        // the skipped NextN layers come right before the output layer, put them on the same device as the output layer
-        // so that the layers assigned by the model loader match the layers above
-        if (hp_nsk > 0 && mparams.n_gpu_layers > 0) {
-            mparams.n_gpu_layers += hp_nsk;
-            if (nd > 1) {
-                // the last devices have no layers if not even the output layer fits
-                size_t id_last = nd - 1;
-                while (ngl_per_device[id_last].n_layer == 0) {
-                    id_last--;
-                }
-                tensor_split[id_last] += hp_nsk;
-            }
-        }
-
         mparams.tensor_split = tensor_split;
 
         size_t itbo = 0;
@@ -695,7 +594,7 @@ static void common_params_fit_impl(
         set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
 
         dmds_t dmd_nl = common_get_device_memory_data_impl(
-            path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+            path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
         add_extra_memory(dmd_nl);
 
         LOG_TRC("%s: memory for test allocation by device:\n", func_name);
@@ -724,7 +623,7 @@ static void common_params_fit_impl(
 
         LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
         dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
-            path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
+            path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
         add_extra_memory(dmds_cpu_moe);
 
         for (size_t id = 0; id < nd; id++) {
@@ -1004,7 +903,8 @@ enum common_params_fit_status common_fit_params(
     return status;
 }
 
-std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const struct llama_context * ctx) {
+void common_memory_breakdown_print(const struct llama_context * ctx) {
+    //const auto & devices = ctx->get_model().devices;
     const auto * model = llama_get_model(ctx);
 
     std::vector<ggml_backend_dev_t> devices;
@@ -1014,6 +914,18 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
 
     llama_memory_breakdown memory_breakdown = llama_get_memory_breakdown(ctx);
 
+    std::vector<std::array<std::string, 9>> table_data;
+    table_data.reserve(devices.size());
+
+    // same data as the table below, for --log-jsonl consumers
+    common_json rows = common_json::array();
+    const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
+    const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
+    const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
+
+    table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
+
+    constexpr size_t MiB = 1024 * 1024;
     const std::vector<std::string> desc_prefixes_strip = {"NVIDIA ", "GeForce ", "Tesla ", "AMD ", "Radeon ", "Instinct "};
 
     // track seen buffer types to avoid double counting:
@@ -1052,130 +964,104 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const stru
         }
     }
 
-    std::vector<common_memory_breakdown_row> rows;
-    rows.reserve(devices.size() + 1 + memory_breakdown.size());
-
+    // print memory breakdown for each device:
     for (size_t i = 0; i < devices.size(); i++) {
         ggml_backend_dev_t dev = devices[i];
+        llama_memory_breakdown_data mb = mb_dev[i];
 
-        common_memory_breakdown_row row;
-        row.kind        = common_memory_breakdown_row::DEVICE;
-        row.name        = ggml_backend_dev_name(dev);
-        row.description = ggml_backend_dev_description(dev);
+        const std::string name = ggml_backend_dev_name(dev);
+        std::string desc = ggml_backend_dev_description(dev);
         for (const std::string & prefix : desc_prefixes_strip) {
-            if (row.description.length() >= prefix.length() && row.description.substr(0, prefix.length()) == prefix) {
-                row.description = row.description.substr(prefix.length());
+            if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {
+                desc = desc.substr(prefix.length());
             }
         }
 
         size_t free, total;
         ggml_backend_dev_memory(dev, &free, &total);
-        row.total   = total;
-        row.free    = free;
-        row.model   = mb_dev[i].model;
-        row.context = mb_dev[i].context;
-        row.compute = mb_dev[i].compute;
-        rows.push_back(std::move(row));
+
+        const size_t self = mb.model + mb.context + mb.compute;
+        const int64_t unaccounted = static_cast<int64_t>(total) - static_cast<int64_t>(free) - static_cast<int64_t>(self);
+
+        table_data.push_back({
+            template_gpu,
+            "  - " + name + " (" + desc + ")",
+            std::to_string(total / MiB),
+            std::to_string(free / MiB),
+            std::to_string(self / MiB),
+            std::to_string(mb.model / MiB),
+            std::to_string(mb.context / MiB),
+            std::to_string(mb.compute / MiB),
+            std::to_string(unaccounted / static_cast<int64_t>(MiB))});
+
+        rows.push_back({
+            {"kind",        "device"},
+            {"name",        name},
+            {"description", desc},
+            {"total",       total / MiB},
+            {"free",        free / MiB},
+            {"self",        self / MiB},
+            {"model",       mb.model / MiB},
+            {"context",     mb.context / MiB},
+            {"compute",     mb.compute / MiB},
+            {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
+        });
     }
 
+    // print memory breakdown for host:
     {
-        common_memory_breakdown_row row;
-        row.kind    = common_memory_breakdown_row::HOST;
-        row.name    = "Host";
-        row.model   = mb_host.model;
-        row.context = mb_host.context;
-        row.compute = mb_host.compute;
-        rows.push_back(std::move(row));
+        const size_t self = mb_host.model + mb_host.context + mb_host.compute;
+        table_data.push_back({
+            template_other,
+            "  - Host",
+            "", // total
+            "", // free
+            std::to_string(self / MiB),
+            std::to_string(mb_host.model / MiB),
+            std::to_string(mb_host.context / MiB),
+            std::to_string(mb_host.compute / MiB),
+            ""}); // unaccounted
+
+        rows.push_back({
+            {"kind",    "host"},
+            {"name",    "Host"},
+            {"self",    self / MiB},
+            {"model",   mb_host.model / MiB},
+            {"context", mb_host.context / MiB},
+            {"compute", mb_host.compute / MiB},
+        });
     }
 
-    // all remaining buffer types:
+    // print memory breakdown for all remaining buffer types:
     for (const auto & buft_mb : memory_breakdown) {
         ggml_backend_buffer_type_t          buft = buft_mb.first;
         const llama_memory_breakdown_data & mb   = buft_mb.second;
         if (seen_buffer_types.count(buft) == 1) {
             continue;
         }
-        common_memory_breakdown_row row;
-        row.kind    = common_memory_breakdown_row::BUFFER_TYPE;
-        row.name    = ggml_backend_buft_name(buft);
-        row.model   = mb.model;
-        row.context = mb.context;
-        row.compute = mb.compute;
-        rows.push_back(std::move(row));
-
-        seen_buffer_types.insert(buft);
-    }
-
-    return rows;
-}
-
-void common_memory_breakdown_print(const struct llama_context * ctx) {
-    const std::vector<common_memory_breakdown_row> rows = common_memory_breakdown_rows(ctx);
-
-    std::vector<std::array<std::string, 9>> table_data;
-    table_data.reserve(rows.size() + 1);
-
-    // same data as the table below, for --log-jsonl consumers
-    common_json json_rows = common_json::array();
-    const std::string template_header = "%s: | %s | %s   %s    %s   %s   %s   %s    %s |\n";
-    const std::string template_gpu    = "%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\n";
-    const std::string template_other  = "%s: | %s | %s   %s    %s = %s + %s + %s    %s |\n";
-
-    table_data.push_back({template_header, "memory breakdown [MiB]", "total", "free", "self", "model", "context", "compute", "unaccounted"});
-
-    constexpr size_t MiB = 1024 * 1024;
-
-    for (const auto & row : rows) {
-        const size_t self = row.model + row.context + row.compute;
-
-        if (row.kind == common_memory_breakdown_row::DEVICE) {
-            const int64_t unaccounted = row.total - row.free - static_cast<int64_t>(self);
-
-            table_data.push_back({
-                template_gpu,
-                "  - " + row.name + " (" + row.description + ")",
-                std::to_string(row.total / MiB),
-                std::to_string(row.free / MiB),
-                std::to_string(self / MiB),
-                std::to_string(row.model / MiB),
-                std::to_string(row.context / MiB),
-                std::to_string(row.compute / MiB),
-                std::to_string(unaccounted / static_cast<int64_t>(MiB))});
-
-            json_rows.push_back({
-                {"kind",        "device"},
-                {"name",        row.name},
-                {"description", row.description},
-                {"total",       row.total / MiB},
-                {"free",        row.free / MiB},
-                {"self",        self / MiB},
-                {"model",       row.model / MiB},
-                {"context",     row.context / MiB},
-                {"compute",     row.compute / MiB},
-                {"unaccounted", unaccounted / static_cast<int64_t>(MiB)},
-            });
-            continue;
-        }
-
+        const std::string name = ggml_backend_buft_name(buft);
+        const size_t self = mb.model + mb.context + mb.compute;
         table_data.push_back({
             template_other,
-            "  - " + row.name,
+            "  - " + name,
             "", // total
             "", // free
             std::to_string(self / MiB),
-            std::to_string(row.model / MiB),
-            std::to_string(row.context / MiB),
-            std::to_string(row.compute / MiB),
+            std::to_string(mb.model / MiB),
+            std::to_string(mb.context / MiB),
+            std::to_string(mb.compute / MiB),
             ""}); // unaccounted
 
-        json_rows.push_back({
-            {"kind",    row.kind == common_memory_breakdown_row::HOST ? "host" : "buffer_type"},
-            {"name",    row.name},
+        rows.push_back({
+            {"kind",    "buffer_type"},
+            {"name",    name},
             {"self",    self / MiB},
-            {"model",   row.model / MiB},
-            {"context", row.context / MiB},
-            {"compute", row.compute / MiB},
+            {"model",   mb.model / MiB},
+            {"context", mb.context / MiB},
+            {"compute", mb.compute / MiB},
         });
+
+        seen_buffer_types.insert(buft);
     }
 
     for (size_t j = 1; j < table_data[0].size(); j++) {
@@ -1195,7 +1081,7 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
 
     LOG_JSON("fit_memory_breakdown", common_json({
         {"unit", "MiB"},
-        {"rows", json_rows},
+        {"rows", rows},
     }));
 }
 
@@ -1205,11 +1091,10 @@ void common_fit_print(
         llama_context_params * cparams) {
     std::vector<ggml_backend_dev_t> devs;
     uint32_t hp_ngl = 0; // hparams.n_gpu_layers
-    uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
     uint32_t hp_nct = 0; // hparams.n_ctx_train
     uint32_t hp_nex = 0; // hparams.n_expert
 
-    auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
+    auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
     GGML_ASSERT(dmd.size() == devs.size() + 1);
 
     for (size_t id = 0; id < devs.size(); id++) {

common/fit.h

@@ -3,7 +3,6 @@
 #include "ggml.h"
 #include "llama.h"
 
-#include <string>
 #include <vector>
 
 enum common_params_fit_status {
@@ -46,47 +45,6 @@ void common_fit_print(
 
 void common_memory_breakdown_print(const llama_context * ctx);
 
-// one row of the memory breakdown: a device, the host or another buffer type
-struct common_memory_breakdown_row {
-    enum kind_t { DEVICE, HOST, BUFFER_TYPE };
-
-    kind_t      kind = DEVICE;
-    std::string name;        // device or buffer type name, "Host" for the host row
-    std::string description; // device description, empty otherwise
-    int64_t     total = 0;   // device memory in bytes, 0 unless kind == DEVICE
-    int64_t     free  = 0;   // free device memory in bytes, 0 unless kind == DEVICE
-    size_t      model   = 0; // bytes held by this context, by kind of buffer
-    size_t      context = 0;
-    size_t      compute = 0;
-};
-
-// memory a model needs at the min and max context --fit may pick
-struct common_memory_reports {
-    uint32_t n_ctx_train = 0;
-    uint32_t n_ctx_max   = 0; // the largest context that fits the free device memory right now
-    uint32_t n_ctx_min   = 0; // --fit-ctx (the smallest context --fit may pick), with the placement set in params
-    std::vector<common_memory_breakdown_row> info_max; // breakdown at n_ctx_max
-    std::vector<common_memory_breakdown_row> info_min; // breakdown at n_ctx_min
-};
-
-// rows of common_memory_breakdown_print: model devices first, then host
-std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llama_context * ctx);
-
-// load model + context with no_alloc and return the memory breakdown, params are used as-is
-// n_ctx_train and n_ctx are optional outputs; throws on failure
-std::vector<common_memory_breakdown_row> common_fit_measure(
-                         const char * path_model,
-           const llama_model_params * mparams,
-         const llama_context_params * cparams,
-                           uint32_t * n_ctx_train,
-                           uint32_t * n_ctx,
-                     ggml_log_level   log_level);
-
-struct common_params;
-
-// measure min and max without allocating, params may be modified as with --fit; throws on failure
-common_memory_reports common_measure_memory(common_params & params);
-
 struct common_device_memory_data {
     int64_t total;
     int64_t free;
@@ -104,7 +62,6 @@ common_device_memory_data_vec common_get_device_memory_data(
          const llama_context_params * cparams,
     std::vector<ggml_backend_dev_t> & devs,
                            uint32_t & hp_ngl,
-                           uint32_t & hp_ngl_skip,
                            uint32_t & hp_n_ctx_train,
                            uint32_t & hp_n_expert,
                      ggml_log_level   log_level);

common/hf-cache.cpp

@@ -2,7 +2,6 @@
 
 #include "build-info.h"
 #include "common.h"
-#include "file-lock.h"
 #include "log.h"
 #include "http.h"
 #include "json.h"
@@ -45,7 +44,7 @@ static fs::path get_cache_directory() {
             {HOME_DIR,                fs::path(".cache") / "huggingface" / "hub"}
         };
         for (const auto & entry : entries) {
-            if (fs::path base = common_get_env(entry.var); !base.empty()) {
+            if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) {
                 return entry.path.empty() ? base : base / entry.path;
             }
         }
@@ -63,7 +62,7 @@ static fs::path get_cache_directory() {
 }
 
 std::string get_cache_path() {
-    return get_cache_directory().string();
+    return fs_path_to_utf8(get_cache_directory());
 }
 
 static std::string folder_name_to_repo(const std::string & folder) {
@@ -162,14 +161,6 @@ static bool is_valid_oid(const std::string & oid) {
     return is_hex_string(oid, 40) || is_hex_string(oid, 64);
 }
 
-std::string get_lock_path(const std::string & repo_id, const std::string & oid) {
-    if (!is_valid_repo_id(repo_id) || !is_valid_oid(oid)) {
-        return {};
-    }
-    fs::path lock_path = fs::path(get_cache_path()) / ".locks" / repo_to_folder_name(repo_id) / (oid + ".lock");
-    return lock_path.string();
-}
-
 static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) {
     if (subpath.is_absolute()) {
         return false; // never do a / b with b absolute
@@ -242,7 +233,7 @@ static std::string get_repo_commit(const std::string & repo_id,
                 LOG_WRN("%s: skip invalid commit: %s\n", __func__, _commit.c_str());
                 continue;
             }
-            const fs::path candidate = _name;
+            const fs::path candidate = fs::u8path(_name);
 
             if (!is_valid_subpath(refs_path, candidate)) {
                 LOG_WRN("%s: skip invalid branch: %s\n", __func__, _name.c_str());
@@ -317,7 +308,7 @@ hf_files get_repo_files(const std::string & repo_id,
             file.repo_id = repo_id;
             file.path = item["path"].get<std::string>();
 
-            const fs::path subpath = file.path;
+            const fs::path subpath = fs::u8path(file.path);
 
             if (!is_valid_subpath(commit_path, subpath)) {
                 LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str());
@@ -340,11 +331,11 @@ hf_files get_repo_files(const std::string & repo_id,
             file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
 
             fs::path final_path = commit_path / subpath;
-            file.final_path = final_path.string();
+            file.final_path = fs_path_to_utf8(final_path);
 
             if (!file.oid.empty() && !fs::exists(final_path)) {
                 fs::path local_path = blobs_path / file.oid;
-                file.local_path = local_path.string();
+                file.local_path = fs_path_to_utf8(local_path);
             } else {
                 file.local_path = file.final_path;
             }
@@ -411,7 +402,7 @@ hf_files get_cached_files(const std::string & repo_id) {
         if (!fs::exists(snapshots_path)) {
             continue;
         }
-        std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
+        std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
 
         if (!is_valid_repo_id(_repo_id)) {
             continue;
@@ -434,8 +425,9 @@ hf_files get_cached_files(const std::string & repo_id) {
             if (!path.empty()) {
                 hf_file file;
                 file.repo_id = _repo_id;
-                file.path = path.generic_string();
-                file.local_path = entry.path().string();
+                const auto generic_path = path.generic_u8string();
+                file.path = std::string(generic_path.begin(), generic_path.end());
+                file.local_path = fs_path_to_utf8(entry.path());
                 file.final_path = file.local_path;
                 files.push_back(std::move(file));
             }
@@ -445,12 +437,12 @@ hf_files get_cached_files(const std::string & repo_id) {
     return files;
 }
 
-std::string finalize_file(const hf_file & file, const std::function<bool()> & keep_waiting) {
+std::string finalize_file(const hf_file & file) {
     static std::atomic<bool> symlinks_disabled{false};
 
     std::error_code ec;
-    fs::path local_path = file.local_path;
-    fs::path final_path = file.final_path;
+    fs::path local_path = fs::u8path(file.local_path);
+    fs::path final_path = fs::u8path(file.final_path);
 
     if (local_path == final_path || fs::exists(final_path, ec)) {
         return file.final_path;
@@ -460,24 +452,6 @@ std::string finalize_file(const hf_file & file, const std::function<bool()> & ke
         return file.final_path;
     }
 
-    // the lock file is shared with huggingface_hub: another process may be
-    // finalizing the same blob
-    common_file_lock lock(get_lock_path(file.repo_id, file.oid));
-    if (!lock.acquire(keep_waiting)) {
-        // without the lock, another process could move the blob while we check it
-        throw std::runtime_error("failed to lock cache for finalization: " + file.final_path);
-    }
-
-    // acquire() only polls keep_waiting while waiting for a busy lock
-    if (keep_waiting && !keep_waiting()) {
-        throw std::runtime_error("download cancelled before finalization: " + file.final_path);
-    }
-
-    // re-check under the lock: the winner may have created the link already
-    if (fs::exists(final_path, ec)) {
-        return file.final_path;
-    }
-
     fs::create_directories(final_path.parent_path(), ec);
 
     if (!symlinks_disabled) {
@@ -515,7 +489,7 @@ bool remove_cached_repo(const std::string & repo_id) {
     std::error_code ec;
     auto removed = fs::remove_all(repo_path, ec);
     if (ec) {
-        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
+        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
         return false;
     }
     return removed > 0;

common/hf-cache.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <functional>
 #include <string>
 #include <vector>
 
@@ -27,14 +26,8 @@ hf_files get_repo_files(
 
 hf_files get_cached_files(const std::string & repo_id = {});
 
-// Lock file path for a blob, shared with the hf CLI: <cache>/.locks/<repo folder>/<oid>.lock
-// Pure path computation: creates nothing. Returns an empty string on invalid repo or oid.
-std::string get_lock_path(const std::string & repo_id, const std::string & oid);
-
-// Create snapshot path (link or move/copy) and return it.
-// Throws when the lock cannot be taken or keep_waiting returns false.
-// A completed snapshot (final_path present) is returned without the lock, so a read-only cache works.
-std::string finalize_file(const hf_file & file, const std::function<bool()> & keep_waiting = {});
+// Create snapshot path (link or move/copy) and return it
+std::string finalize_file(const hf_file & file);
 
 // Remove the entire cached directory for a repo, returns true if removed
 bool remove_cached_repo(const std::string & repo_id);

common/jinja/caps.cpp

@@ -6,9 +6,7 @@
 // we can remove it in the future when we figure out a better way to define inputs using jinja::value
 #include "json.h"
 
-#include <algorithm>
 #include <functional>
-#include <optional>
 #include <sstream>
 
 #define FILENAME "jinja-caps"
@@ -186,10 +184,6 @@ caps caps_get(jinja::program & prog) {
                     {
                         {"role", "user"},
                         {"content", json::array({
-                            {
-                                {"type", "text"},
-                                {"text", content_marker}
-                            }
                         })}
                     }
                 });
@@ -590,58 +584,6 @@ caps caps_get(jinja::program & prog) {
         }
     );
 
-    if (result.supports_reasoning_effort) {
-        JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort levels");
-
-        static const std::vector<std::string> candidates = { "minimal", "low", "medium", "high", "xhigh", "max" };
-
-        // renders a single user turn with the given effort, nothing when the template raises
-        const auto render = [&](const std::string & effort) {
-            std::optional<std::string> rendered;
-            caps_try_execute(
-                prog,
-                [&]() {
-                    // messages
-                    return json::array({
-                        {
-                            {"role", "user"},
-                            {"content", "User message"}
-                        },
-                    });
-                },
-                [&](context & ctx) {
-                    ctx.set_val("enable_thinking", mk_val<value_bool>(true));
-                    caps_apply_reasoning_effort(ctx, effort);
-                },
-                nullptr, // tools_fn
-                [&](context &, bool success, value &, value &, const std::string & output) {
-                    if (success) {
-                        rendered = output;
-                    }
-                }
-            );
-            return rendered;
-        };
-
-        // an unknown level shows the render of the template fallback, a level that renders the same is not honored
-        // levels sharing a render are aliases and the highest one is kept
-        const auto unknown = render("REASONING_EFFORT_UNKNOWN");
-        std::vector<std::string> renders;
-        for (const auto & level : candidates) {
-            const auto rendered = render(level);
-            if (!rendered || rendered == unknown) {
-                continue;
-            }
-            const auto it = std::find(renders.begin(), renders.end(), *rendered);
-            if (it != renders.end()) {
-                result.reasoning_effort_levels[it - renders.begin()] = level;
-            } else {
-                renders.push_back(*rendered);
-                result.reasoning_effort_levels.push_back(level);
-            }
-        }
-    }
-
     JJ_DEBUG("%s\n", result.to_string().c_str());
 
     return result;

common/jinja/caps.h

@@ -4,7 +4,6 @@
 
 #include <string>
 #include <map>
-#include <vector>
 
 namespace jinja {
 
@@ -20,9 +19,6 @@ struct caps {
     // supports reasoning effort levels
     bool supports_reasoning_effort = false;
 
-    // reasoning effort levels the template renders distinctly, from lowest to highest
-    std::vector<std::string> reasoning_effort_levels;
-
     // one of the 2 content capabilities must be true
     bool supports_string_content = true;
     bool supports_typed_content = false;

common/jinja/lexer.cpp

@@ -1,9 +1,7 @@
 #include "lexer.h"
 #include "runtime.h"
-#include "unicode.h"
 
 #include <cctype>
-#include <charconv>
 #include <functional>
 #include <map>
 #include <string>
@@ -77,24 +75,6 @@ lexer_result lexer::tokenize(const std::string & source) {
                 }
                 // add escaped char
                 char escaped_char = src[pos++];
-                if (escaped_char == 'x') {
-                    if (pos + 1 >= src.size()) {
-                        throw lexer_exception("incomplete hexadecimal escape", source, pos);
-                    }
-
-                    unsigned int value = 0;
-                    const char * first = &src[pos];
-                    const char * last  = first + 2;
-
-                    auto [ptr, ec] = std::from_chars(first, last, value, 16);
-                    if (ec != std::errc{} || ptr != last) {
-                        throw lexer_exception("invalid hexadecimal escape", source, pos);
-                    }
-
-                    str += common_unicode_cpt_to_utf8(value);
-                    pos += 2;
-                    continue;
-                }
                 if (escape_chars.find(escaped_char) == escape_chars.end()) {
                     throw lexer_exception(std::string("unknown escape character \\") + escaped_char, source, pos);
                 }

common/jinja/parser.cpp

@@ -21,17 +21,6 @@ static bool is_type(const statement_ptr & ptr) {
 class parser {
     const std::vector<token> & tokens;
     size_t current = 0;
-    int depth = 0;
-
-    struct depth_guard {
-        parser & p;
-        depth_guard(parser & p) : p(p) {
-            if (++p.depth > JINJA_RECURSIVE_LIMIT) {
-                throw parser_exception("Parser Error: template is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep", p.source, p.peek().pos);
-            }
-        }
-        ~depth_guard() { p.depth--; }
-    };
 
     std::string source; // for error reporting
 
@@ -105,7 +94,6 @@ private:
     }
 
     statement_ptr parse_any() {
-        depth_guard guard(*this);
         size_t start_pos = current;
         switch (peek().t) {
             case token::comment:
@@ -266,7 +254,6 @@ private:
             size_t pos0 = current;
             ++current; // consume {%
             ++current; // consume 'elif'
-            depth_guard guard(*this);
             alternate.push_back(parse_if_statement(pos0)); // nested If
         } else if (is_statement({"else"})) {
             ++current; // consume {%
@@ -343,7 +330,6 @@ private:
     }
 
     statement_ptr parse_if_expression() {
-        depth_guard guard(*this);
         auto a = parse_logical_or_expression();
         if (is_identifier("if")) {
             // Ternary expression
@@ -387,7 +373,6 @@ private:
     statement_ptr parse_logical_negation_expression() {
         // Try parse unary operators
         if (is_identifier("not")) {
-            depth_guard guard(*this);
             size_t start_pos = current;
             auto op = next();
             return mk_stmt<unary_expression>(start_pos, op, parse_logical_negation_expression());
@@ -473,7 +458,6 @@ private:
 
     statement_ptr parse_unary_expression() {
         if (is(token::unary_operator)) {
-            depth_guard guard(*this);
             size_t start_pos = current;
             auto op = next();
             return mk_stmt<unary_expression>(start_pos, op, parse_unary_expression());
@@ -493,11 +477,9 @@ private:
         size_t start_pos = current;
         auto expr = mk_stmt<call_expression>(start_pos, std::move(callee), parse_args());
         auto member = parse_member_expression(std::move(expr)); // foo.x().y
-        if (!is(token::open_paren)) {
-            return member;
-        }
-        depth_guard guard(*this);
-        return parse_call_expression(std::move(member)); // foo.x()()
+        return is(token::open_paren)
+            ? parse_call_expression(std::move(member)) // foo.x()()
+            : std::move(member);
     }
 
     statements parse_args() {

common/jinja/runtime.cpp

@@ -3,7 +3,6 @@
 #include "value.h"
 #include "utils.h"
 
-#include <exception>
 #include <string>
 #include <vector>
 #include <memory>
@@ -53,39 +52,32 @@ static void ensure_key_type_allowed(const value & val) {
 
 // execute with error handling
 value statement::execute(context & ctx) const {
-    std::exception_ptr pending;
-    std::string message;
     try {
         return execute_impl(ctx);
     } catch (const continue_statement::signal & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const break_statement::signal & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const rethrown_exception & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const not_implemented_exception & /* ex */) {
-        pending = std::current_exception();
+        throw;
     } catch (const std::exception & e) {
         const std::string & source = *ctx.src;
         if (source.empty()) {
             std::ostringstream oss;
             oss << "\nError executing " << type() << " at position " << pos << ": " << e.what();
-            message = oss.str();
+            throw rethrown_exception(oss.str());
         } else {
             std::ostringstream oss;
             oss << "\n------------\n";
             oss << "While executing " << type() << " at " << get_line_col(source, pos) << " in source:\n";
             oss << peak_source(source, pos) << "\n";
             oss << "Error: " << e.what();
-            message = oss.str();
+            // throw as another exception to avoid repeated formatting
+            throw rethrown_exception(oss.str());
         }
     }
-    // throw outside the catch: MSVC keeps the unwound frames on the stack while a catch block runs
-    if (pending) {
-        std::rethrow_exception(pending);
-    }
-    // throw as another exception to avoid repeated formatting
-    throw rethrown_exception(message);
 }
 
 value identifier::execute_impl(context & ctx) const {
@@ -307,13 +299,58 @@ value binary_expression::execute_impl(context & ctx) const {
     throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
 }
 
+static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) {
+    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
+    if (ctx.is_get_stats) {
+        value_t::stats_t::mark_used(input);
+        input->stats.ops.insert(name);
+    }
+    auto builtins = input->get_builtins();
+    auto it = builtins.find(name);
+    if (it != builtins.end()) {
+        JJ_DEBUG("Binding built-in '%s'", name.c_str());
+        return mk_val<value_func>(name, it->second, input);
+    }
+    if (undef_on_missing) {
+        return mk_val<value_undefined>(name);
+    }
+    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
+}
+
 static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
     JJ_DEBUG("Applying filter to %s", input->type().c_str());
 
+    auto set_filter_alias = [](auto & filter_id) {
+        if (filter_id == "count") {
+            filter_id = "length";
+        } else if (filter_id == "d") {
+            filter_id = "default";
+        } else if (filter_id == "e") {
+            filter_id = "escape";
+        } else if (filter_id == "trim") {
+            filter_id = "strip";
+        }
+    };
+
     if (is_stmt<identifier>(filter)) {
         auto filter_id = cast_stmt<identifier>(filter)->val;
+
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str());
-        return apply_filter_by_name(ctx, filter_id, input, func_args(ctx));
+        // TODO: Refactor filters so this coercion can be done automatically
+        if (!input->is_undefined() && !is_val<value_string>(input) && (
+            filter_id == "capitalize" ||
+            filter_id == "lower" ||
+            filter_id == "replace" ||
+            filter_id == "strip" ||
+            filter_id == "title" ||
+            filter_id == "upper" ||
+            filter_id == "wordcount"
+        )) {
+            JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
+            input = mk_val<value_string>(input->as_string());
+        }
+        return try_builtin_func(ctx, filter_id, input)->invoke(func_args(ctx));
 
     } else if (is_stmt<call_expression>(filter)) {
         auto call = cast_stmt<call_expression>(filter);
@@ -322,13 +359,14 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
         }
         auto filter_id = cast_stmt<identifier>(call->callee)->val;
 
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
         func_args args(ctx);
         for (const auto & arg_expr : call->args) {
             args.push_back(arg_expr->execute(ctx));
         }
 
-        return apply_filter_by_name(ctx, filter_id, input, args);
+        return try_builtin_func(ctx, filter_id, input)->invoke(args);
 
     } else {
         throw std::runtime_error("Invalid filter expression");
@@ -709,11 +747,7 @@ value macro_statement::execute_impl(context & ctx) const {
     std::string name = cast_stmt<identifier>(this->name)->val;
 
     const func_handler func = [this, name](const func_args & args) -> value {
-        if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
-            throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
-        }
         context macro_ctx(args.ctx); // new scope for macro execution
-        macro_ctx.macro_depth = args.ctx.macro_depth + 1;
 
         bind_parameters(name, this->args, args, macro_ctx);
 
@@ -744,11 +778,7 @@ value call_statement::execute_impl(context & ctx) const {
     context caller_ctx(ctx); // new scope for caller execution
 
     const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value {
-        if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
-            throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
-        }
         context block_ctx(caller_ctx); // new scope for block execution
-        block_ctx.macro_depth = args.ctx.macro_depth + 1;
 
         bind_parameters("caller", this->caller_args, args, block_ctx);
 

common/jinja/runtime.h

@@ -58,8 +58,6 @@ struct context {
 
     bool is_get_stats = false; // whether to collect stats
 
-    int macro_depth = 0;
-
     visitor_fn visitor;
 
     // src is optional, used for error reporting
@@ -80,7 +78,6 @@ struct context {
         src(parent.src),
         current_time(parent.current_time),
         is_get_stats(parent.is_get_stats),
-        macro_depth(parent.macro_depth),
         // inherit variables (for example, when entering a new scope)
         env(mk_val<value_object>(*parent.env)) {}
 
@@ -713,7 +710,7 @@ struct rethrown_exception : public std::exception {
 
 //////////////////////
 
-static void gather_string_parts_recursive(const value & val, value_string & parts, int depth = 0) {
+static void gather_string_parts_recursive(const value & val, value_string & parts) {
     // TODO: probably allow print value_none as "None" string? currently this breaks some templates
     if (is_val<value_string>(val)) {
         const auto & str_val = cast_val<value_string>(val)->val_str;
@@ -721,12 +718,10 @@ static void gather_string_parts_recursive(const value & val, value_string & part
     } else if (is_val<value_int>(val) || is_val<value_float>(val) || is_val<value_bool>(val)) {
         std::string str_val = val->as_string().str();
         parts->val_str.append(str_val);
-    } else if (is_val<value_array>(val) && depth >= JINJA_RECURSIVE_LIMIT) {
-        throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
     } else if (is_val<value_array>(val)) {
         auto items = cast_val<value_array>(val)->as_array();
         for (const auto & item : items) {
-            gather_string_parts_recursive(item, parts, depth + 1);
+            gather_string_parts_recursive(item, parts);
         }
     }
 }

common/jinja/utils.h

@@ -6,8 +6,6 @@
 #include <cstdint>
 #include <cstring>
 
-#define JINJA_RECURSIVE_LIMIT 128
-
 namespace jinja {
 
 static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {

common/jinja/value.cpp

@@ -6,7 +6,6 @@
 #include "json.h"
 
 #include <sstream>
-#include <stdexcept>
 #include <string>
 #include <cctype>
 #include <cfloat>
@@ -72,17 +71,6 @@ const std::vector<value> & func_args::get_args() const {
     return args;
 }
 
-void func_args::ensure_kwargs(std::initializer_list<const char *> names) const {
-    for (const auto & arg : args) {
-        if (is_val<value_kwarg>(arg)) {
-            auto * kwarg = cast_val<value_kwarg>(arg);
-            if (std::find(names.begin(), names.end(), kwarg->key) == names.end()) {
-                throw raised_exception("Function '" + func_name + "' got an unexpected keyword argument '" + kwarg->key + "'");
-            }
-        }
-    }
-}
-
 /**
  * Function that mimics Python's array slicing.
  */
@@ -145,14 +133,6 @@ static value empty_value_fn(const func_args &) {
         return mk_val<T>();
     }
 }
-
-template<typename T>
-static value empty_value_no_args_fn(const func_args & args) {
-    args.ensure_kwargs();
-    args.ensure_count(1, 1);
-    return empty_value_fn<T>(args);
-}
-
 template<typename T>
 static value test_type_fn(const func_args & args) {
     args.ensure_count(1);
@@ -319,53 +299,74 @@ static value & get_attribute(const value & val, const value & attr, value & defa
     return default_val;
 }
 
-template<bool is_reject, bool lookup_attr>
-static value select_or_reject(const func_args & args) {
-    // example: array | select("odd") -> test_is_odd(item)
-    // example: array | selectattr("status", "equalto", "active") -> test_is_equalto(item.status, "active")
-    constexpr size_t test_pos = lookup_attr ? 2 : 1;
-    args.ensure_count(test_pos);
-    args.ensure_val<value_array>(args.get_pos(0));
-    value attr;
-    if constexpr (lookup_attr) {
-        attr = args.get_pos(1);
-        args.ensure_val<value_string>(attr);
-    }
-
-    auto & builtins = global_builtins();
-    auto test_it = builtins.end();
-    if (args.count() > test_pos) {
-        args.ensure_val<value_string>(args.get_pos(test_pos));
-        std::string test_name = args.get_pos(test_pos)->as_string().str();
-        test_it = builtins.find("test_is_" + test_name);
-        if (test_it == builtins.end()) {
-            throw raised_exception(args.func_name + ": unknown test '" + test_name + "'");
-        }
-    }
+template<bool is_reject>
+static value selectattr(const func_args & args) {
+    args.ensure_count(2, 4);
+    args.ensure_vals<value_array, value_string, value_string, value_string>(true, true, false, false);
 
     auto arr = args.get_pos(0)->as_array();
+    auto attribute = args.get_pos(1);
     auto out = mk_val<value_array>();
     value val_default = mk_val<value_undefined>();
 
-    for (const auto & item : arr) {
-        value val = item;
-        if constexpr (lookup_attr) {
-            val = get_attribute(item, attr, val_default);
+    if (args.count() == 2) {
+        // example: array | selectattr("active")
+        for (const auto & item : arr) {
+            value attr_val = get_attribute(item, attribute, val_default);
+            bool is_selected = attr_val->as_bool();
+            if constexpr (is_reject) is_selected = !is_selected;
+            if (is_selected) out->push_back(item);
         }
-        bool is_selected;
-        if (test_it == builtins.end()) {
-            is_selected = val->as_bool();
-        } else {
+        return out;
+
+    } else if (args.count() == 3) {
+        // example: array | selectattr("equalto", "text")
+        // translated to: test_is_equalto(item, "text")
+        std::string test_name = args.get_pos(1)->as_string().str();
+        value test_val = args.get_pos(2);
+        auto & builtins = global_builtins();
+        auto it = builtins.find("test_is_" + test_name);
+        if (it == builtins.end()) {
+            throw raised_exception("selectattr: unknown test '" + test_name + "'");
+        }
+        auto test_fn = it->second;
+        for (const auto & item : arr) {
             func_args test_args(args.ctx);
-            test_args.push_back(val);
-            for (size_t i = test_pos + 1; i < args.count(); i++) {
-                test_args.push_back(args.get_pos(i));
-            }
-            is_selected = test_it->second(test_args)->as_bool();
+            test_args.push_back(item); // current object
+            test_args.push_back(test_val); // extra argument
+            value test_result = test_fn(test_args);
+            bool is_selected = test_result->as_bool();
+            if constexpr (is_reject) is_selected = !is_selected;
+            if (is_selected) out->push_back(item);
         }
-        if constexpr (is_reject) is_selected = !is_selected;
-        if (is_selected) out->push_back(item);
+        return out;
+
+    } else if (args.count() == 4) {
+        // example: array | selectattr("status", "equalto", "active")
+        // translated to: test_is_equalto(item.status, "active")
+        std::string test_name = args.get_pos(2)->as_string().str();
+        auto extra_arg = args.get_pos(3);
+        auto & builtins = global_builtins();
+        auto it = builtins.find("test_is_" + test_name);
+        if (it == builtins.end()) {
+            throw raised_exception("selectattr: unknown test '" + test_name + "'");
+        }
+        auto test_fn = it->second;
+        for (const auto & item : arr) {
+            value attr_val = get_attribute(item, attribute, val_default);
+            func_args test_args(args.ctx);
+            test_args.push_back(attr_val); // attribute value
+            test_args.push_back(extra_arg); // extra argument
+            value test_result = test_fn(test_args);
+            bool is_selected = test_result->as_bool();
+            if constexpr (is_reject) is_selected = !is_selected;
+            if (is_selected) out->push_back(item);
+        }
+        return out;
+    } else {
+        throw raised_exception("selectattr: invalid number of arguments");
     }
+
     return out;
 }
 
@@ -424,12 +425,7 @@ const func_builtins & global_builtins() {
             throw raised_exception("Jinja Exception: " + msg);
         }},
         {"dict", toobject},
-        {"namespace", [](const func_args & args) -> value {
-            // unlike a dict, a namespace has no methods: ns.items is the attribute, as in jinja2
-            value out = toobject(args);
-            cast_val<value_object>(out)->has_builtins = false;
-            return out;
-        }},
+        {"namespace", toobject},
         {"strftime_now", [](const func_args & args) -> value {
             args.ensure_vals<value_string>();
             std::string format = args.get_pos(0)->as_string().str();
@@ -613,52 +609,6 @@ const func_builtins & global_builtins() {
     return builtins;
 }
 
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) {
-    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
-    if (ctx.is_get_stats) {
-        value_t::stats_t::mark_used(input);
-        input->stats.ops.insert(name);
-    }
-    auto builtins = input->get_builtins();
-    auto it = builtins.find(name);
-    if (it != builtins.end()) {
-        JJ_DEBUG("Binding built-in '%s'", name.c_str());
-        return mk_val<value_func>(name, it->second, input);
-    }
-    if (undef_on_missing) {
-        return mk_val<value_undefined>(name);
-    }
-    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
-}
-
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args) {
-    if (filter_id == "count") {
-        filter_id = "length";
-    } else if (filter_id == "d") {
-        filter_id = "default";
-    } else if (filter_id == "e") {
-        filter_id = "escape";
-    } else if (filter_id == "trim") {
-        filter_id = "strip";
-    }
-
-    // TODO: Refactor filters so this coercion can be done automatically
-    if (!input->is_undefined() && !is_val<value_string>(input) && (
-        filter_id == "capitalize" ||
-        filter_id == "lower" ||
-        filter_id == "replace" ||
-        filter_id == "strip" ||
-        filter_id == "title" ||
-        filter_id == "upper" ||
-        filter_id == "wordcount"
-    )) {
-        JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
-        input = mk_val<value_string>(input->as_string());
-    }
-
-    return try_builtin_func(ctx, filter_id, input)->invoke(args);
-}
-
 
 const func_builtins & value_int_t::get_builtins() const {
     static const func_builtins builtins = {
@@ -732,22 +682,16 @@ const func_builtins & value_string_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
         {"upper", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().uppercase();
             return mk_val<value_string>(str);
         }},
         {"lower", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().lowercase();
             return mk_val<value_string>(str);
         }},
         {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
             value val_input = args.get_pos(0);
             if (!is_val<value_string>(val_input)) {
                 throw raised_exception("strip() first argument must be a string");
@@ -778,15 +722,11 @@ const func_builtins & value_string_t::get_builtins() const {
             }
         }},
         {"title", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().titlecase();
             return mk_val<value_string>(str);
         }},
         {"capitalize", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().capitalize();
             return mk_val<value_string>(str);
@@ -1088,10 +1028,6 @@ const func_builtins & value_bool_t::get_builtins() const {
     return builtins;
 }
 
-[[noreturn]] static value array_sum_not_implemented(const func_args &) {
-    throw not_implemented_exception("Array sum builtin not implemented");
-}
-
 const func_builtins & value_array_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
@@ -1154,13 +1090,11 @@ const func_builtins & value_array_t::get_builtins() const {
             auto arr = slice(val->as_array(), start, stop, step);
             return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(arr)) : mk_val<value_array>(std::move(arr));
         }},
-        {"selectattr", select_or_reject<false, true>},
-        {"select", select_or_reject<false, false>},
-        {"rejectattr", select_or_reject<true, true>},
-        {"reject", select_or_reject<true, false>},
-        {"sum", array_sum_not_implemented},
+        {"selectattr", selectattr<false>},
+        {"select", selectattr<false>},
+        {"rejectattr", selectattr<true>},
+        {"reject", selectattr<true>},
         {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
             args.ensure_count(1, 3);
             if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("join() first argument must be an array");
@@ -1198,28 +1132,13 @@ const func_builtins & value_array_t::get_builtins() const {
         {"tojson", tojson},
         {"map", [](const func_args & args) -> value {
             args.ensure_count(2);
-            value val = args.get_pos(0);
-            if (!is_val<value_array>(val)) {
+            if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("map: first argument must be an array");
             }
             if (!is_val<value_kwarg>(args.get_args().at(1))) {
-                value filter = args.get_pos(1);
-                if (!is_val<value_string>(filter)) {
-                    throw raised_exception("map: filter name must be a string");
-                }
-                auto out = mk_val<value_array>();
-                func_args filter_args(args.ctx);
-                const auto & all_args = args.get_args();
-                for (size_t i = 2; i < all_args.size(); ++i) {
-                    filter_args.push_back(all_args[i]);
-                }
-                const std::string filter_id = filter->as_string().str();
-                auto arr = val->as_array();
-                for (const auto & item : arr) {
-                    out->push_back(apply_filter_by_name(args.ctx, filter_id, item, filter_args));
-                }
-                return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+                throw not_implemented_exception("map: filter-mapping not implemented");
             }
+            value val       = args.get_pos(0);
             value attribute = args.get_kwarg_or_pos("attribute", 1);
             value default_val = args.get_kwarg("default", mk_val<value_undefined>());
             auto out = mk_val<value_array>();
@@ -1228,7 +1147,7 @@ const func_builtins & value_array_t::get_builtins() const {
                 value attr_val = get_attribute(item, attribute, default_val);
                 out->push_back(attr_val);
             }
-            return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+            return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(out->as_array())) : out;
         }},
         {"append", [](const func_args & args) -> value {
             args.ensure_count(2);
@@ -1501,18 +1420,14 @@ const func_builtins & value_none_t::get_builtins() const {
 const func_builtins & value_undefined_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
-        {"capitalize", empty_value_no_args_fn<value_string>},
+        {"capitalize", empty_value_fn<value_string>},
         {"first", empty_value_fn<value_undefined>},
         {"items", empty_value_fn<value_array>},
-        {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
-            args.ensure_count(1, 3);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"join", empty_value_fn<value_string>},
         {"last", empty_value_fn<value_undefined>},
         {"length", empty_value_fn<value_int>},
         {"list", empty_value_fn<value_array>},
-        {"lower", empty_value_no_args_fn<value_string>},
+        {"lower", empty_value_fn<value_string>},
         {"map", empty_value_fn<value_array>},
         {"max", empty_value_fn<value_undefined>},
         {"min", empty_value_fn<value_undefined>},
@@ -1525,16 +1440,12 @@ const func_builtins & value_undefined_t::get_builtins() const {
         {"selectattr", empty_value_fn<value_array>},
         {"sort", empty_value_fn<value_array>},
         {"string", empty_value_fn<value_string>},
-        {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"strip", empty_value_fn<value_string>},
         {"sum", empty_value_fn<value_int>},
-        {"title", empty_value_no_args_fn<value_string>},
+        {"title", empty_value_fn<value_string>},
         {"truncate", empty_value_fn<value_string>},
         {"unique", empty_value_fn<value_array>},
-        {"upper", empty_value_no_args_fn<value_string>},
+        {"upper", empty_value_fn<value_string>},
         {"wordcount", empty_value_fn<value_int>},
     };
     return builtins;
@@ -1544,8 +1455,7 @@ const func_builtins & value_undefined_t::get_builtins() const {
 //////////////////////////////////
 
 
-// the runtime walks values with recursion, a deeper input overflows the stack
-static value from_json(const common_json & j, bool mark_input, int depth = 0) {
+static value from_json(const common_json & j, bool mark_input) {
     if (j.is_null()) {
         return mk_val<value_none>();
     } else if (j.is_boolean()) {
@@ -1560,18 +1470,16 @@ static value from_json(const common_json & j, bool mark_input, int depth = 0) {
             str->mark_input();
         }
         return str;
-    } else if ((j.is_array() || j.is_object()) && depth >= JINJA_RECURSIVE_LIMIT) {
-        throw std::invalid_argument("JSON is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
     } else if (j.is_array()) {
         auto arr = mk_val<value_array>();
         for (const auto & item : j) {
-            arr->push_back(from_json(item, mark_input, depth + 1));
+            arr->push_back(from_json(item, mark_input));
         }
         return arr;
     } else if (j.is_object()) {
         auto obj = mk_val<value_object>();
         for (auto it = j.begin(); it != j.end(); ++it) {
-            obj->insert(it.key(), from_json(it.value(), mark_input, depth + 1));
+            obj->insert(it.key(), from_json(it.value(), mark_input));
         }
         return obj;
     } else {
@@ -1766,8 +1674,6 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
             }
         }
         oss << "\"";
-    } else if ((is_val<value_array>(val) || is_val<value_object>(val)) && curr_lvl >= JINJA_RECURSIVE_LIMIT) {
-        throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
     } else if (is_val<value_array>(val)) {
         const auto & arr = val->as_array();
         oss << "[";
@@ -1833,20 +1739,17 @@ std::string value_to_string_repr(const value & val) {
 }
 
 // stats utility
-void value_t::stats_t::mark_used(value & val, bool deep, int depth) {
+void value_t::stats_t::mark_used(value & val, bool deep) {
     val->stats.used = true;
-    if (deep && (is_val<value_array>(val) || is_val<value_object>(val)) && depth >= JINJA_RECURSIVE_LIMIT) {
-        throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
-    }
     if (deep) {
         if (is_val<value_array>(val)) {
             for (auto & item : val->val_arr) {
-                mark_used(item, deep, depth + 1);
+                mark_used(item, deep);
             }
         } else if (is_val<value_object>(val)) {
             for (auto & pair : val->val_obj) {
-                mark_used(pair.first, deep, depth + 1);
-                mark_used(pair.second, deep, depth + 1);
+                mark_used(pair.first, deep);
+                mark_used(pair.second, deep);
             }
         }
     }

common/jinja/value.h

@@ -7,7 +7,6 @@
 #include <cmath>
 #include <cstdint>
 #include <functional>
-#include <initializer_list>
 #include <map>
 #include <memory>
 #include <set>
@@ -120,7 +119,7 @@ struct value_t {
         // ops can be builtin calls or operators: "array_access", "object_access"
         std::set<std::string> ops;
         // utility to recursively mark value and its children as used
-        static void mark_used(value & val, bool deep = false, int depth = 0);
+        static void mark_used(value & val, bool deep = false);
     } stats;
 
     value_t() = default;
@@ -177,8 +176,6 @@ protected:
 //
 
 const func_builtins & global_builtins();
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
 
 std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
 
@@ -661,7 +658,6 @@ public:
     size_t count() const { return args.size(); }
     void push_back(const value & val);
     void push_front(const value & val);
-    void ensure_kwargs(std::initializer_list<const char *> names = {}) const;
     void ensure_count(size_t min, size_t max = 999) const {
         size_t n = args.size();
         if (n < min || n > max) {

common/json-schema-to-grammar.cpp

@@ -272,12 +272,15 @@ static bool is_reserved_name(const std::string & name) {
     return RESERVED_NAMES.find(name) != RESERVED_NAMES.end();
 }
 
+static std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+");
 static std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"\\\\]");
 static std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]");
 static std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
     {'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}, {'\\', "\\\\"}
 };
 
+static const int MAX_PATTERN_DEPTH = 100;
+
 static std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?', '^', '$'};
 static std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'^', '$', '.', '[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};
 
@@ -309,22 +312,6 @@ static std::string format_literal(const std::string & literal) {
 
 std::string gbnf_format_literal(const std::string & literal) { return format_literal(literal); }
 
-// same as std::regex_replace with [^a-zA-Z0-9-]+ -> "-", no std::regex: libstdc++ recurses per char
-std::string gbnf_format_rule_name(const std::string & name) {
-    std::string out;
-    bool in_invalid = false;
-    for (char c : name) {
-        if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-') {
-            out += c;
-            in_invalid = false;
-        } else if (!in_invalid) {
-            out += '-';
-            in_invalid = true;
-        }
-    }
-    return out;
-}
-
 static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
     if (pos + 1 >= pattern.length() || pattern[pos] != '\\') {
         return 0;
@@ -360,17 +347,6 @@ private:
     std::unordered_set<std::string> _refs_being_resolved;
     std::vector<std::string> _errors;
     std::vector<std::string> _warnings;
-    int _depth = 0;
-
-    struct depth_guard {
-        int & depth;
-        depth_guard(int & depth) : depth(depth) {
-            if (++depth > COMMON_RECURSIVE_LIMIT) {
-                throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
-            }
-        }
-        ~depth_guard() { depth--; }
-    };
 
     template <typename T>
     static const T & as(const common_chat_schema & node) {
@@ -378,7 +354,7 @@ private:
     }
 
     std::string _add_rule(const std::string & name, const std::string & rule) {
-        std::string esc_name = gbnf_format_rule_name(name);
+        std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
         if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
             _rules[esc_name] = rule;
             return esc_name;
@@ -506,7 +482,7 @@ private:
                         }
                     }
                     paren_depth++;
-                    if (paren_depth > COMMON_RECURSIVE_LIMIT) {
+                    if (paren_depth > MAX_PATTERN_DEPTH) {
                         throw unsupported_pattern("pattern nesting too deep");
                     }
                     seq.emplace_back("(" + to_rule(transform()) + ")", false);
@@ -668,11 +644,7 @@ private:
         std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
         std::ostringstream out;
         out << "[\"] ( ";
-        // one level per char of the longest name
-        std::function<void(size_t, int)> visit = [&](size_t idx, int depth) {
-            if (depth >= COMMON_RECURSIVE_LIMIT) {
-                throw std::invalid_argument("JSON schema property name is longer than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " chars");
-            }
+        std::function<void(size_t)> visit = [&](size_t idx) {
             const auto & node = trie.nodes[idx];
             std::string rejects;
             auto first = true;
@@ -687,7 +659,7 @@ private:
                 out << "[" << c << "]";
                 if (!trie.nodes[child].children.empty()) {
                     out << " (";
-                    visit(child, depth + 1);
+                    visit(child);
                     out << ")";
                 } else {
                     out << " " << char_rule << "+";
@@ -697,7 +669,7 @@ private:
                 out << " | [^\"" << rejects << "] " << char_rule << "*";
             }
         };
-        visit(0, 0);
+        visit(0);
 
         out << " )";
         if (trie.nodes[0].pattern < 0) {
@@ -710,7 +682,8 @@ private:
     std::string _resolve_ref(const common_chat_schema_ref & schema) {
         auto it = schema.ref.find('#');
         std::string ref_fragment = it != std::string::npos ? schema.ref.substr(it + 1) : schema.ref;
-        std::string ref_name = "ref" + gbnf_format_rule_name(ref_fragment);
+        static const std::regex nonalphanumeric_regex(R"([^a-zA-Z0-9-]+)");
+        std::string ref_name = "ref" + std::regex_replace(ref_fragment, nonalphanumeric_regex, "-");
         if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(schema.ref) == _refs_being_resolved.end()) {
             if (!schema.target) {
                 _errors.push_back("Unresolved $ref " + schema.ref);
@@ -860,13 +833,10 @@ public:
         std::unordered_set<std::string> required;
         std::vector<std::pair<std::string, const common_chat_schema *>> properties;
         std::map<std::string, size_t> enum_values;
-        std::function<void(const common_chat_schema &, bool, int)> add_component = [&](const common_chat_schema & comp, bool is_required, int depth) {
-            if (depth >= COMMON_RECURSIVE_LIMIT) {
-                throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
-            }
+        std::function<void(const common_chat_schema &, bool)> add_component = [&](const common_chat_schema & comp, bool is_required) {
             if (comp.kind() == common_chat_schema::KIND_REF) {
                 if (const auto * target = as<common_chat_schema_ref>(comp).target) {
-                    add_component(*target, is_required, depth + 1);
+                    add_component(*target, is_required);
                 }
             } else if (comp.kind() == common_chat_schema::KIND_OBJECT) {
                 for (const auto & prop : as<common_chat_schema_object>(comp).properties) {
@@ -884,10 +854,10 @@ public:
         for (const auto & child : schema.children) {
             if (child->kind() == common_chat_schema::KIND_ANY_OF) {
                 for (const auto & alt : as<common_chat_schema_any_of>(*child).children) {
-                    add_component(*alt, false, 0);
+                    add_component(*alt, false);
                 }
             } else {
-                add_component(*child, true, 0);
+                add_component(*child, true);
             }
         }
         if (!enum_values.empty()) {
@@ -905,8 +875,6 @@ public:
     }
 
     std::string visit(const common_chat_schema & schema, const std::string & name) {
-        // the $ref targets are visited too, a $ref chain is not bounded by the builder
-        depth_guard guard(_depth);
         std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
         std::string sub_name  = name + (name.empty() ? "" : "-");
 

common/json-schema-to-grammar.h

@@ -19,6 +19,5 @@ struct common_grammar_options {
 };
 
 std::string gbnf_format_literal(const std::string & literal);
-std::string gbnf_format_rule_name(const std::string & name);
 
 std::string build_grammar(const std::function<void(const common_grammar_builder &)> & cb, const common_grammar_options & options = {});

common/json-schema.cpp

@@ -9,18 +9,6 @@
 #include <utility>
 #include <vector>
 
-namespace {
-struct depth_guard {
-    int & depth;
-    depth_guard(int & depth) : depth(depth) {
-        if (++depth > COMMON_RECURSIVE_LIMIT) {
-            throw std::invalid_argument("JSON schema is nested more than " + std::to_string(COMMON_RECURSIVE_LIMIT) + " levels deep");
-        }
-    }
-    ~depth_guard() { depth--; }
-};
-} // namespace
-
 class common_chat_schema_builder {
     const common_json &      root_;
     common_chat_schema_document & doc_;
@@ -236,14 +224,7 @@ class common_chat_schema_builder {
         return node;
     }
 
-    int depth_ = 0;
-
     common_chat_schema_ptr build_node(const common_json & schema, const std::string & path) {
-        depth_guard guard(depth_);
-        return build_node_impl(schema, path);
-    }
-
-    common_chat_schema_ptr build_node_impl(const common_json & schema, const std::string & path) {
         if (!schema.is_object()) {
             fail(path, "schema must be an object");
         }
@@ -384,9 +365,7 @@ static common_chat_schema::value_type json_type(const common_json & value) {
     return common_chat_schema::TYPE_OBJECT;
 }
 
-// depth counts the $ref, anyOf and allOf hops, a $ref chain is not bounded by the builder
-static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited, int & depth) {
-    depth_guard guard(depth);
+static common_chat_schema::type_set value_types_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
     switch (s.kind()) {
         case common_chat_schema::KIND_ANY:
             return common_chat_schema::type_set::all();
@@ -420,21 +399,21 @@ static common_chat_schema::type_set value_types_impl(const common_chat_schema &
                 // a cycle contributes no type, to be safe
                 return {};
             }
-            auto types = value_types_impl(*target, visited, depth);
+            auto types = value_types_impl(*target, visited);
             visited.erase(target);
             return types;
         }
         case common_chat_schema::KIND_ANY_OF: {
             common_chat_schema::type_set types;
             for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                types |= value_types_impl(*child, visited, depth);
+                types |= value_types_impl(*child, visited);
             }
             return types;
         }
         case common_chat_schema::KIND_ALL_OF: {
             auto types = common_chat_schema::type_set::all();
             for (const auto & child : static_cast<const common_chat_schema_all_of &>(s).children) {
-                types &= value_types_impl(*child, visited, depth);
+                types &= value_types_impl(*child, visited);
             }
             return types;
         }
@@ -444,12 +423,10 @@ static common_chat_schema::type_set value_types_impl(const common_chat_schema &
 
 common_chat_schema::type_set common_chat_schema::value_types() const {
     std::unordered_set<const common_chat_schema *> visited;
-    int depth = 0;
-    return value_types_impl(*this, visited, depth);
+    return value_types_impl(*this, visited);
 }
 
-static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited, int & depth) {
-    depth_guard guard(depth);
+static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<const common_chat_schema *> & visited) {
     switch (s.kind()) {
         case common_chat_schema::KIND_STRING:
             return true;
@@ -468,13 +445,13 @@ static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<
             if (!target || !visited.insert(target).second) {
                 return false;
             }
-            bool result = may_be_string_impl(*target, visited, depth);
+            bool result = may_be_string_impl(*target, visited);
             visited.erase(target);
             return result;
         }
         case common_chat_schema::KIND_ANY_OF:
             for (const auto & child : static_cast<const common_chat_schema_any_of &>(s).children) {
-                if (may_be_string_impl(*child, visited, depth)) {
+                if (may_be_string_impl(*child, visited)) {
                     return true;
                 }
             }
@@ -486,7 +463,7 @@ static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<
                 if (child->kind() == common_chat_schema::KIND_ANY) {
                     continue;
                 }
-                if (!may_be_string_impl(*child, visited, depth)) {
+                if (!may_be_string_impl(*child, visited)) {
                     return false;
                 }
                 any_string = true;
@@ -500,8 +477,7 @@ static bool may_be_string_impl(const common_chat_schema & s, std::unordered_set<
 
 bool common_chat_schema::may_be_string() const {
     std::unordered_set<const common_chat_schema *> visited;
-    int depth = 0;
-    return may_be_string_impl(*this, visited, depth);
+    return may_be_string_impl(*this, visited);
 }
 
 const char * common_chat_schema::kind_name(node_kind kind) {

common/parsers/deepseek.cpp

@@ -1,5 +1,4 @@
 #include "parsers.h"
-#include "log.h"
 
 // The DeepSeek V4 reference implementation renders consecutive tool results into a single
 // user block, ordered by the tool call order of the preceding assistant message (matched
@@ -76,30 +75,13 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
     // non-thinking generation prompt ends with a bare </think> instead of an empty
     // <think></think> pair.
     const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
-    // V4.1: DSML tag names carry a leading space, e.g. <｜DSML｜ calls>
-    const bool is_v41 = is_v4 && tmpl.source().find("｜DSML｜ calls>") != std::string::npos;
 
     std::optional<json> adjusted_messages;
     if (is_v4) {
         adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
     }
 
-    // V4.1 can declare tools on a system message instead of top-level "tools": parse calls to those too.
-    json tools = inputs.tools;
-    if (is_v41 && !(tools.is_array() && !tools.empty()) && inputs.messages.is_array()) {
-        for (const auto & msg : inputs.messages) {
-            if (msg.value("role", "") == "system" && msg.contains("tools") && msg.at("tools").is_array()) {
-                if (!tools.is_array()) {
-                    tools = json::array();
-                }
-                for (const auto & tool : msg.at("tools")) {
-                    tools.push_back(tool);
-                }
-            }
-        }
-    }
-
-    auto has_tools           = tools.is_array() && !tools.empty();
+    auto has_tools           = inputs.tools.is_array() && !inputs.tools.empty();
     auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
     auto extract_reasoning   = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
     auto include_grammar     = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
@@ -108,39 +90,17 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
     if (is_v4 && has_response_format) {
         additional_context = json{ { "response_format", inputs.json_schema } };
     }
-    // V4.1 raises on anything but low/high/max or 1-100: map minimal -> low, xhigh -> max, medium -> default
-    if (is_v41 && inputs.extra_context.is_object() && inputs.extra_context.contains("reasoning_effort") &&
-        inputs.extra_context.at("reasoning_effort").is_string()) {
-        const std::string effort = inputs.extra_context.at("reasoning_effort").get<std::string>();
-        json mapped;
-        if (effort == "minimal") {
-            mapped = "low";
-        } else if (effort == "xhigh") {
-            mapped = "max";
-        } else if (effort == "medium") {
-            mapped = nullptr;
-        }
-        if (effort == "minimal" || effort == "xhigh" || effort == "medium") {
-            LOG_DBG("%s: DeepSeek V4.1 reasoning_effort '%s' -> %s\n", __func__, effort.c_str(),
-                    mapped.is_null() ? "unset (template default)" : mapped.get<std::string>().c_str());
-            if (!additional_context.has_value()) {
-                additional_context = json::object();
-            }
-            (*additional_context)["reasoning_effort"] = mapped;
-        }
-    }
 
     const std::string DSML         = "｜DSML｜";
     const std::string THINK_START  = "<think>";
     const std::string THINK_END    = "</think>";
-    const std::string TAG_SP       = is_v41 ? " " : "";
-    const std::string TC_BLOCK     = is_v41 ? "calls" : is_v4 ? "tool_calls" : "function_calls";
-    const std::string FC_START     = "<" + DSML + TAG_SP + TC_BLOCK + ">";
-    const std::string FC_END       = "</" + DSML + TAG_SP + TC_BLOCK + ">";
-    const std::string INVOKE_START = "<" + DSML + TAG_SP + "invoke";
-    const std::string INVOKE_END   = "</" + DSML + TAG_SP + "invoke>";
-    const std::string PARAM_START  = "<" + DSML + TAG_SP + "parameter";
-    const std::string PARAM_END    = "</" + DSML + TAG_SP + "parameter>";
+    const std::string TC_BLOCK     = is_v4 ? "tool_calls" : "function_calls";
+    const std::string FC_START     = "<" + DSML + TC_BLOCK + ">";
+    const std::string FC_END       = "</" + DSML + TC_BLOCK + ">";
+    const std::string INVOKE_START = "<" + DSML + "invoke";
+    const std::string INVOKE_END   = "</" + DSML + "invoke>";
+    const std::string PARAM_START  = "<" + DSML + "parameter";
+    const std::string PARAM_END    = "</" + DSML + "parameter>";
     const std::string GEN_PROMPT   = "<｜Assistant｜>";
     const std::string TC_SEPARATOR = "\n\n";
 
@@ -192,7 +152,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
         // build tool call section first since we might need it in reasoning
         auto tool_choice = p.choice();
         if (has_tool_calls) {
-            foreach_function(tools, [&](size_t tool_index, const json & tool) {
+            foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
                 const auto & function = tool.at("function");
                 std::string  name     = function.at("name");
 
@@ -263,10 +223,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
         }
 
         if (extract_reasoning && inputs.enable_thinking) {
-            // V4.1 renders a continuation without reasoning as "<｜Assistant｜></think>" even in thinking mode,
-            // so consume the bare THINK_END too or it leaks into the content
-            reasoning = is_v41 ? p.optional((THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END) | p.literal(THINK_END))
-                               : p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
+            reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
             reasoning_with_tc = THINK_START +
                 p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
                 p.space() + obligatory_tool_calls;

common/parsers/gpt-oss.cpp

@@ -71,7 +71,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
         auto content         = p.rule("message-content", p.until("<|end|>"));
         auto channel         = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
         auto constrain_type  = p.chars("[A-Za-z0-9_-]", 1, -1);
-        auto constraint      = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
 
         // Occasionally, gpt-oss-20b will prefix channels with this commentary
         auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));
@@ -85,7 +84,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
 
         auto analysis = p.ref("analysis");
         auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);
-        auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(content));
+        auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));
 
         // Consume any unsolicited tool calls, e.g. builtin functions
         auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));
@@ -93,6 +92,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
         auto any = p.rule("any", preamble | analysis);
 
         if (has_response_format) {
+            auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
             auto response_format = p.rule("response-format",
                 p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
                 p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
@@ -109,6 +109,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
                 const auto   params   = common_chat_tool_parameters(function);
 
                 auto func_name  = p.literal(" to=functions.") + p.tool_name(p.literal(name));
+                auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
                 auto args       = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params));
 
                 // recipient in role header

common/parsers/qwen3-coder.cpp

@@ -74,24 +74,15 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
     data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
         auto generation_prompt = p.literal(GEN_PREFIX);
 
-        // The template strips the newline before </think>, so leave it out of the reasoning
-        auto think_body  = p.until_one_of({ "\n</think>", "</think>", "<tool_call>" });
-        auto think_close = p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"));
-
-        auto reasoning_extracted = p.optional("<think>" + p.space() + p.reasoning(think_body) + think_close);
-        auto reasoning_skipped   = p.optional("<think>" + p.space() + think_body + think_close);
-
         auto reasoning = p.eps();
         if (supports_reasoning && extract_reasoning) {
-            reasoning = reasoning_extracted;
+            reasoning = p.optional("<think>" + p.space() +
+                                   p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
+                                   (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
         }
 
         // Response format parser
         if (has_response_format) {
-            // The answer must be bare JSON, so the think block is consumed even when it is not extracted
-            if (supports_reasoning && !extract_reasoning) {
-                reasoning = reasoning_skipped;
-            }
             return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
         }
 
@@ -175,15 +166,8 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
             auto calls      = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
             auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
 
-            // The templates separate content from the tool calls with one or two newlines,
-            // stop the content before them
-            std::vector<std::string> content_ends;
-            for (const auto & start : tool_call_starts) {
-                content_ends.insert(content_ends.end(), { "\n\n" + start, "\n" + start, start });
-            }
-
             return generation_prompt +
-                   (reasoning << p.content(p.until_one_of(content_ends)) << tool_calls);
+                   (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
         }
 
         // Content only parser

common/peg-parser.cpp

@@ -10,12 +10,10 @@
 #include <initializer_list>
 #include <map>
 #include <memory>
+#include <regex>
 #include <set>
 #include <stdexcept>
 
-// The parser can run with a lower limit than COMMON_RECURSIVE_LIMIT
-static constexpr int RULE_RECURSIVE_LIMIT = 64;
-
 // Trick to catch missing branches
 template <typename T>
 inline constexpr bool is_always_false_v = false;
@@ -188,17 +186,13 @@ std::string common_peg_ast_node::sanitized_text() const {
 }
 
 void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const {
-    // no recursion: the AST is several nodes deep per level of the input
-    std::vector<common_peg_ast_id> stack = { id };
-    while (!stack.empty()) {
-        const auto cur = stack.back();
-        stack.pop_back();
-        if (cur == COMMON_PEG_INVALID_AST_ID) {
-            continue;
-        }
-        const auto & node = get(cur);
-        visitor(node);
-        stack.insert(stack.end(), node.children.rbegin(), node.children.rend());
+    if (id == COMMON_PEG_INVALID_AST_ID) {
+        return;
+    }
+    const auto & node = get(id);
+    visitor(node);
+    for (const auto & child : node.children) {
+        visit(child, visitor);
     }
 }
 
@@ -721,9 +715,8 @@ struct parser_executor {
                 return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
             }
 
-            if (match == common_trie::PARTIAL_MATCH && !ctx.is_complete()) {
-                // Found a partial match extending to end of input, return everything before it.
-                // On a complete input the delimiter can no longer form, so the bytes belong to the text
+            if (match == common_trie::PARTIAL_MATCH) {
+                // Found a partial match extending to end of input, return everything before it
                 return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
             }
 
@@ -743,15 +736,8 @@ struct parser_executor {
     }
 
     common_peg_parse_result operator()(const common_peg_rule_parser & p) {
-        // only rules recurse, a recursive rule is entered once per input level
-        if (ctx.rule_depth[p.child] >= RULE_RECURSIVE_LIMIT) {
-            throw common_peg_recursion_error("input is nested more than " + std::to_string(RULE_RECURSIVE_LIMIT) + " levels deep");
-        }
-
         // Parse the child
-        ctx.rule_depth[p.child]++;
         auto result = arena.parse(p.child, ctx, start_pos);
-        ctx.rule_depth[p.child]--;
 
         if (!result.fail()) {
             std::string_view text;
@@ -833,10 +819,6 @@ common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx,
     if (root_ == COMMON_PEG_INVALID_PARSER_ID) {
         throw std::runtime_error("No root parser set");
     }
-
-    // Initialize rule depth for all parsers
-    ctx.rule_depth.assign(parsers_.size(), 0);
-
     return parse(root_, ctx, start);
 }
 
@@ -1087,6 +1069,11 @@ common_peg_parser operator|(const std::string & str, const common_peg_parser & p
     return operator|(str.c_str(), p);
 }
 
+static std::string rule_name(const std::string & name) {
+    static const std::regex invalid_rule_chars_re("[^a-zA-Z0-9-]+");
+    return std::regex_replace(name, invalid_rule_chars_re, "-");
+}
+
 common_peg_parser_builder::common_peg_parser_builder() {}
 
 common_peg_parser common_peg_parser_builder::sequence(const std::vector<common_peg_parser_id> & parsers) {
@@ -1168,14 +1155,14 @@ common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p,
 }
 
 common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
-    auto clean_name = gbnf_format_rule_name(name);
+    auto clean_name = rule_name(name);
     auto rule_id = arena_.add_parser(common_peg_rule_parser{clean_name, p.id(), trigger});
     arena_.add_rule(clean_name, rule_id);
     return ref(clean_name);
 }
 
 common_peg_parser common_peg_parser_builder::rule(const std::string & name, const std::function<common_peg_parser()> & builder_fn, bool trigger) {
-    auto clean_name = gbnf_format_rule_name(name);
+    auto clean_name = rule_name(name);
     if (arena_.has_rule(clean_name)) {
         return ref(clean_name);
     }

common/peg-parser.h

@@ -6,7 +6,6 @@
 
 #include <memory>
 #include <set>
-#include <stdexcept>
 #include <unordered_map>
 #include <string>
 #include <string_view>
@@ -162,10 +161,9 @@ struct common_peg_parse_result {
 };
 
 enum common_peg_parse_flags {
-    COMMON_PEG_PARSE_FLAG_NONE     = 0,
-    COMMON_PEG_PARSE_FLAG_LENIENT  = 1 << 0,
-    COMMON_PEG_PARSE_FLAG_DEBUG    = 1 << 1,
-    COMMON_PEG_PARSE_FLAG_COMPLETE = 1 << 2,
+    COMMON_PEG_PARSE_FLAG_NONE    = 0,
+    COMMON_PEG_PARSE_FLAG_LENIENT = 1 << 0,
+    COMMON_PEG_PARSE_FLAG_DEBUG   = 1 << 1,
 };
 
 inline common_peg_parse_flags operator|(common_peg_parse_flags a, common_peg_parse_flags b) {
@@ -184,10 +182,6 @@ inline common_peg_parse_flags operator~(common_peg_parse_flags a) {
     return static_cast<common_peg_parse_flags>(~int(a));
 }
 
-struct common_peg_recursion_error : std::invalid_argument {
-    using std::invalid_argument::invalid_argument;
-};
-
 struct common_peg_parse_context {
     std::string input;               // [h,   e,  l,  l,  o,  _,  w,  o,  r,  l,  d]
     std::vector<llama_token> tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1]
@@ -195,7 +189,6 @@ struct common_peg_parse_context {
     common_peg_ast_arena ast;
 
     int parse_depth;
-    std::vector<int> rule_depth; // index: rule child id
 
     common_peg_parse_context(common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)
         : flags(flags), parse_depth(0) {}
@@ -210,7 +203,6 @@ struct common_peg_parse_context {
 
     bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }
     bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }
-    bool is_complete() const { return flags & COMMON_PEG_PARSE_FLAG_COMPLETE; }
 };
 
 class common_peg_arena;

common/preset.cpp

@@ -171,12 +171,12 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     std::map<std::string, std::map<std::string, std::string>> parsed;
 
     if (!std::filesystem::exists(path)) {
-        throw std::runtime_error("preset file does not exist: " + path.string());
+        throw std::runtime_error("preset file does not exist: " + fs_path_to_utf8(path));
     }
 
     std::ifstream file(path);
     if (!file.good()) {
-        throw std::runtime_error("failed to open server preset file: " + path.string());
+        throw std::runtime_error("failed to open server preset file: " + fs_path_to_utf8(path));
     }
 
     std::string contents((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
@@ -225,7 +225,7 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     common_peg_parse_context ctx(contents);
     const auto result = parser.parse(ctx);
     if (!result.success()) {
-        throw std::runtime_error("failed to parse server config file: " + path.string());
+        throw std::runtime_error("failed to parse server config file: " + fs_path_to_utf8(path));
     }
 
     std::string current_section = COMMON_PRESET_DEFAULT_NAME;
@@ -323,7 +323,7 @@ common_presets common_preset_context::load_from_ini(const std::filesystem::path
                 }
                 LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
             } else if (ignore_unknown_keys) {
-                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.string().c_str());
+                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), fs_path_to_utf8(path).c_str());
             } else {
                 throw std::runtime_error(string_format(
                     "option '%s' not recognized in preset '%s'",
@@ -385,7 +385,7 @@ static bool is_draft_file(const std::string & fname) {
 }
 
 common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const {
-    const std::filesystem::path dir = models_dir;
+    const std::filesystem::path dir = std::filesystem::u8path(models_dir);
     if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) {
         throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str()));
     }
@@ -401,7 +401,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
             if (!entry.is_regular_file(ec)) {
                 continue;
             }
-            const std::string fname = entry.path().filename().string();
+            const std::string fname = fs_path_to_utf8(entry.path().filename());
             if (!string_ends_with(fname, ".gguf")) {
                 continue;
             }
@@ -422,9 +422,9 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         if (!path.empty()) {
             models.push_back({
                 /* name        */ name,
-                /* path        */ path.string(),
-                /* path_mmproj */ mmproj_file.string(), // can be empty
-                /* path_draft  */ draft_file.string()   // can be empty
+                /* path        */ fs_path_to_utf8(path),
+                /* path_mmproj */ fs_path_to_utf8(mmproj_file), // can be empty
+                /* path_draft  */ fs_path_to_utf8(draft_file)    // can be empty
             });
         }
     };
@@ -432,13 +432,13 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
     for (const auto & entry : std::filesystem::directory_iterator(dir)) {
         std::error_code ec;
         if (entry.is_directory(ec)) {
-            scan_subdir(entry.path(), entry.path().filename().string());
+            scan_subdir(entry.path(), fs_path_to_utf8(entry.path().filename()));
             continue;
         }
         if (!entry.is_regular_file(ec)) {
             continue;
         }
-        const std::string fname = entry.path().filename().string();
+        const std::string fname = fs_path_to_utf8(entry.path().filename());
         if (!string_ends_with(fname, ".gguf")) {
             continue;
         }
@@ -450,7 +450,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         string_replace_all(name, ".gguf", "");
         models.push_back({
             /* name        */ name,
-            /* path        */ entry.path().string(),
+            /* path        */ fs_path_to_utf8(entry.path()),
             /* path_mmproj */ "",
             /* path_draft  */ ""
         });

common/reasoning-budget.cpp

@@ -136,8 +136,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to
             // track the end sequence within forced_tokens so it is also reported on DONE
             const int32_t match = ctx->end_matcher.advance(token);
             ctx->force_pos++;
-            // an end tag that is already in the prompt (e.g. "<think></think>") also ends forcing
-            if (match >= 0 || ctx->force_pos >= ctx->forced_tokens.size()) {
+            if (ctx->force_pos >= ctx->forced_tokens.size()) {
                 ctx->state = REASONING_BUDGET_DONE;
                 ctx->end_match = match;
                 COM_TRC("%s", "forced sequence complete, done\n");

common/speculative.cpp

@@ -19,7 +19,6 @@
 #include <fstream>
 #include <iomanip>
 #include <map>
-#include <random>
 #include <cinttypes>
 
 #define SPC_DBG(fmt, ...) LOG_DBG("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__)
@@ -1011,10 +1010,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
     bool    is_mrope       = false;
     int32_t selector_top_k = 0;
 
-    // DFlash2 probabilistic drafting: one generator per sequence and the seed it was last seeded with (-1: reseed)
-    std::vector<std::mt19937> dflash2_rng;
-    std::vector<int64_t>      dflash2_seed;
-
     // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout
     const bool is_dspark;
 
@@ -1027,9 +1022,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
     const int32_t * target_layer_ids   = nullptr; // model_dft's extract layer indices
     uint32_t        target_layer_ids_n = 0;
 
-    // inject target features from the target's device buffer instead of host copies (env LLAMA_DFLASH_DEV=0 disables)
-    bool use_dev_features = false;
-
     common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
             common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
         : common_speculative_impl(type, n_seq, params.draft.n_max)
@@ -1068,8 +1060,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
         selector_top_k = llama_model_dflash_selector_top_k(model_dft);
         is_dflash2     = selector_top_k > 0;
-        dflash2_rng.resize(n_seq);
-        dflash2_seed.assign(n_seq, -1);
         char lilicorr_meta[32] = {};
         lilicorr_top_k = llama_model_meta_val_str(model_dft, "lilicorr.candidate_topk", lilicorr_meta, sizeof(lilicorr_meta)) >= 0
                 ? std::atoi(lilicorr_meta) : 0;
@@ -1144,23 +1134,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
         }
 
-        // the device buffer keeps layers in ascending id order, the fc input expects target_layer_ids order
-        {
-            const char * env = getenv("LLAMA_DFLASH_DEV");
-            const bool ascending = std::adjacent_find(target_layer_ids, target_layer_ids + target_layer_ids_n,
-                    [](int32_t a, int32_t b) { return a >= b; }) == target_layer_ids + target_layer_ids_n;
-
-            use_dev_features = !is_lilicorr && ascending && (env == nullptr || atoi(env) != 0);
-            if (use_dev_features) {
-                llama_set_embeddings_layer_inp_dev(ctx_tgt, true);
-                use_dev_features = llama_supports_embd_row(ctx_dft);
-                if (!use_dev_features) {
-                    llama_set_embeddings_layer_inp_dev(ctx_tgt, false);
-                }
-            }
-            LOG_INF("%s: DFlash target features via %s\n", __func__, use_dev_features ? "device buffer" : "host copies");
-        }
-
         // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
         llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr);
         llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
@@ -1185,9 +1158,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             return;
         }
 
-        // reseed per request rather than per round, or two identical requests differ
-        dflash2_seed[seq_id] = -1;
-
         const int32_t N = (int32_t) prompt.size();
         if (N <= 0) {
             return;
@@ -1253,25 +1223,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
                 const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);
 
-                if (use_dev_features) {
-                    // the features are the target batch rows in the target's device buffer
-                    batch_inject.clear();
-                    for (int32_t i = 0; i < n_chunk; ++i) {
-                        const int32_t   row        = i_batch_beg[seq_id] + offset + i;
-                        const llama_pos p          = batch_in.tokens[row].pos[0];
-                        const llama_pos pos_arr[4] = { p, p, p, 0 };
-                        batch_inject.add_embd_row(row, pos_arr, seq_id, false);
-                    }
-
-                    const int32_t rc = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch_inject.get());
-                    if (rc != 0) {
-                        LOG_ERR("%s: llama_process(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
-                                __func__, rc, (int) n_chunk, (int) offset);
-                        return false;
-                    }
-                    continue;
-                }
-
                 // gather target features per extract layer; the fused decode encodes and
                 // injects them into the K/V cache at the target positions
                 features_buf.resize((size_t) n_chunk * n_embd_enc);
@@ -1387,21 +1338,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
                 const float * lattice = llama_get_embeddings_nextn(ctx_dft);
                 GGML_ASSERT(lattice && "DFlash2 selector produced no lattice");
 
-                // DFlash2 can instead sample each step and hand q to the verifier for rejection sampling
-                const bool sample = is_dflash2 && params.probabilistic && dp.result_q;
-
-                // greedy keeps temperature 1, sampling uses the target's
-                const float temp = sample ? dp.temp : 1.0f;
-
-                if (sample && dflash2_seed[seq_id] != dp.seed) {
-                    dflash2_seed[seq_id] = dp.seed;
-                    // the default seed means random; mixed so it differs from the target's, as in spec_retune
-                    dflash2_rng[seq_id].seed(dp.seed == LLAMA_DEFAULT_SEED ? std::random_device{}() : dp.seed ^ 0x85ebca6bu);
-                }
-
-                std::vector<float>   probs;
-                std::vector<int32_t> sorted_idx;
-
                 int32_t predecessor = 0;
                 for (int32_t i = 1; i < (is_dflash2 ? n_block_tokens : params.n_max + 1); ++i) {
                     const float * row = lattice + (size_t) (beg + i) * n_embd_dec;
@@ -1410,53 +1346,21 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
 
                     predecessor = (int32_t) std::distance(scores,
                             std::max_element(scores, scores + top_k));
-                    if (params.p_min > 0.0f || sample) {
-                        // softmax(scores / temp) at the argmax, i.e. 1 / sum(exp((s_k - s_max) / temp))
-                        probs.resize(top_k);
+                    if (params.p_min > 0.0f) {
+                        // softmax(scores) at the argmax, i.e. 1 / sum(exp(s_k - s_max))
                         float sum = 0.0f;
                         for (int32_t k = 0; k < top_k; ++k) {
-                            probs[k] = std::exp((scores[k] - scores[predecessor]) / temp);
-                            sum += probs[k];
+                            sum += std::exp(scores[k] - scores[predecessor]);
                         }
                         if (1.0f / sum < params.p_min) {
                             break;
                         }
-                        for (int32_t k = 0; k < top_k; ++k) {
-                            probs[k] /= sum;
-                        }
-                    }
-                    if (sample) {
-                        // ggml_top_k leaves candidates unordered, so walk them by token id to keep a seed's draft stable
-                        sorted_idx.resize(top_k);
-                        for (int32_t k = 0; k < top_k; ++k) {
-                            sorted_idx[k] = k;
-                        }
-                        std::sort(sorted_idx.begin(), sorted_idx.end(), [row](int32_t a, int32_t b) { return row[a] < row[b]; });
-
-                        // a round-off overshoot keeps the argmax, which has q > 0
-                        double u = std::uniform_real_distribution<double>(0.0, 1.0)(dflash2_rng[seq_id]);
-                        for (const int32_t k : sorted_idx) {
-                            u -= probs[k];
-                            if (u < 0.0) {
-                                predecessor = k;
-                                break;
-                            }
-                        }
-
-                        auto & q = dp.result_q->emplace_back();
-                        q.reserve(top_k);
-                        for (int32_t k = 0; k < top_k; ++k) {
-                            q.push_back({ (llama_token) row[k], scores[k], probs[k] });
-                        }
                     }
                     result.push_back((llama_token) row[predecessor]);
                 }
 
                 if (result.size() < (size_t) params.n_min) {
                     result.clear();
-                    if (sample) {
-                        dp.result_q->clear();
-                    }
                 }
                 continue;
             }
@@ -1613,12 +1517,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             }
         }
 
+        llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ false);
+        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true);
+
         is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt;
         chain_heads   = n_mtp_layers > 1 && !is_mem_shared;
 
-        llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ is_mem_shared);
-        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true);
-
         if (chain_heads) {
             this->params.n_max = std::min(this->params.n_max, n_mtp_layers);
 
@@ -1772,28 +1676,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             }
 
             const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1;
-            if (is_mem_shared && !batch_in.tokens[i_batch_end[seq_id]].output) {
-                verify_h_rows[seq_id] = 0;
-                continue;
-            }
+            verify_h_rows[seq_id] = n_rows;
+            verify_h[seq_id].resize((size_t) n_rows * n_embd);
 
-            const bool all_rows_output = std::all_of(
-                    batch_in.tokens.begin() + i_batch_beg[seq_id],
-                    batch_in.tokens.begin() + i_batch_end[seq_id] + 1,
-                    [](const common_batch::token & token) { return token.output; });
-            const int32_t h_rows = is_mem_shared && !all_rows_output ? 1 : n_rows;
-            const int32_t h_offset = is_mem_shared && !all_rows_output ? i_batch_end[seq_id] : i_batch_beg[seq_id];
-
-            verify_h_rows[seq_id] = h_rows;
-            verify_h[seq_id].resize((size_t) h_rows * n_embd);
-
-            for (int32_t i = 0; i < h_rows; ++i) {
-                const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, h_offset + i);
+            for (int32_t i = 0; i < n_rows; ++i) {
+                const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, i_batch_beg[seq_id] + i);
                 std::memcpy(verify_h[seq_id].data() + (size_t) i * n_embd, h, row_bytes);
             }
 
             std::memcpy(pending_h[seq_id].data(),
-                    verify_h[seq_id].data() + (size_t) (h_rows - 1) * n_embd, row_bytes);
+                    verify_h[seq_id].data() + (size_t) (n_rows - 1) * n_embd, row_bytes);
         }
 
         return true;
@@ -1979,29 +1871,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
         const size_t row_bytes = (size_t) n_embd * sizeof(float);
         std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes);
     }
-
-    bool get_state(llama_seq_id seq_id, std::vector<uint8_t> & data) const override {
-        if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) {
-            return false;
-        }
-
-        const size_t row_bytes = (size_t) n_embd * sizeof(float);
-        data.resize(row_bytes);
-        std::memcpy(data.data(), pending_h[seq_id].data(), row_bytes);
-        return true;
-    }
-
-    void set_state(llama_seq_id seq_id, const std::vector<uint8_t> & data) override {
-        if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) {
-            return;
-        }
-
-        const size_t row_bytes = (size_t) n_embd * sizeof(float);
-        if (data.size() != row_bytes) {
-            return;
-        }
-        std::memcpy(pending_h[seq_id].data(), data.data(), row_bytes);
-    }
 };
 
 // state of self-speculation (simple implementation, not ngram-map)

common/speculative.h

@@ -73,7 +73,7 @@ struct common_speculative_draft_params {
     // the generated draft from the last _draft() call
     llama_tokens * result;
 
-    // candidate distribution per drafted token; set it to make draft-simple, draft-mtp and draft-dflash (DFlash2 only) sample
+    // candidate distribution per drafted token; set it to make draft-simple and draft-mtp sample
     std::vector<std::vector<llama_token_data>> * result_q = nullptr;
 
     // the target's temp and seed, read only when the drafter samples probabilistically

common/win-app.manifest

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
-  <application xmlns="urn:schemas-microsoft-com:asm.v3">
-    <windowsSettings>
-      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
-      <longPathAware xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">true</longPathAware>
-    </windowsSettings>
-  </application>
-</assembly>

conversion/__init__.py

@@ -17,11 +17,8 @@ __all__ = [
 
 
 TEXT_MODEL_MAP: dict[str, str] = {
-    "NEOChatModel": "sensenova_u1",
     "AfmoeForCausalLM": "afmoe",
     "LagunaForCausalLM": "laguna",
-    "RicursiveForCausalLM": "ricursive",
-    "FlintForCausalLM": "flint",
     "ApertusForCausalLM": "llama",
     "ArceeForCausalLM": "llama",
     "ArcticForCausalLM": "arctic",
@@ -46,9 +43,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ChameleonForConditionalGeneration": "chameleon",
     "ChatGLMForConditionalGeneration": "chatglm",
     "ChatGLMModel": "chatglm",
-    "ChatterboxModel": "chatterbox",
     "ClefModel": "clef",
-    "CloverMoeModelLM": "clover",
     "CodeShellForCausalLM": "codeshell",
     "CogVLMForCausalLM": "cogvlm",
     "Cohere2MoeForCausalLM": "command_r",
@@ -63,7 +58,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "DeepseekV2ForCausalLM": "deepseek",
     "DeepseekV3ForCausalLM": "deepseek",
     "DeepseekV32ForCausalLM": "deepseek",
-    "Xing4_0ForCausalLM": "xing",
     "DFlashDraftModel": "qwen",
     "DFlash2DraftModel": "qwen",
     "LiLiCorrDraftModel": "nemotron",
@@ -74,7 +68,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "LingDSparkModel": "qwen",
     "DeepseekV4ForCausalLM": "deepseek",
     "DeepseekV4DSparkModel": "deepseek",
-    "DeepseekV41ForCausalLM": "deepseek41",
     "DistilBertForMaskedLM": "bert",
     "DistilBertForSequenceClassification": "bert",
     "DistilBertModel": "bert",
@@ -97,7 +90,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "FalconForCausalLM": "falcon",
     "FalconH1ForCausalLM": "falcon_h1",
     "FalconMambaForCausalLM": "mamba",
-    "FishOmniModel": "drama3",
     "GPT2LMHeadModel": "gpt2",
     "GPTBigCodeForCausalLM": "starcoder",
     "GPTNeoXForCausalLM": "gptneox",
@@ -177,11 +169,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "D1OmniModel": "lfm2",
     "Lfm2Model": "lfm2",
     "Lfm2MoeForCausalLM": "lfm2",
-    "Lfm3D1Model": "lfm3",
-    "Lfm3ForCausalLM": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
-    "LimiteForCausalLM": "limite",
     "Llama4ForCausalLM": "llama",
     "Llama4ForConditionalGeneration": "llama",
     "LlamaBidirectionalModel": "llama",
@@ -201,15 +188,12 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "MambaLMHeadModel": "mamba",
     "MapleForCausalLM": "maple",
     "MellumForCausalLM": "mellum",
-    "Metis16ForCausalLM": "metis",
     "MiMoV2FlashForCausalLM": "mimo",
     "MiMoV2ForCausalLM": "mimo",
     "MiniCPM3ForCausalLM": "minicpm",
     "MiniCPMForCausalLM": "minicpm",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
-    "MiniCPMOCRForSequenceClassification": "qwen",
-    "MiniCPMOcrForConditionalGeneration": "minicpm",
     "MiniMaxText01ForCausalLM": "minimax",
     "MiniMaxM1ForCausalLM": "minimax",
     "MiniMaxM2ForCausalLM": "minimax",
@@ -219,7 +203,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "Ministral3Model": "mistral3",
     "Mistral3ForConditionalGeneration": "mistral3",
     "Mistral3Model": "mistral3",
-    "MistralLarge4ForConditionalGeneration": "mistral4",
     "VoxtralRealtimeForConditionalGeneration": "mistral3",
     "MistralForCausalLM": "llama",
     "MixtralForCausalLM": "llama",
@@ -228,7 +211,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ModernBertForSequenceClassification": "bert",
     "ModernBertModel": "bert",
     "NanbeigeForCausalLM": "nanbeige",
-    "Nemotron3_5TranscribeForConditionalGeneration": "nemotron",
     "NemotronForCausalLM": "nemotron",
     "NemotronHForCausalLM": "nemotron",
     "NemotronHPuzzleForCausalLM": "nemotron",
@@ -301,7 +283,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "SeedOssForCausalLM": "olmo",
     "SmallThinkerForCausalLM": "smallthinker",
     "SmolLM3ForCausalLM": "llama",
-    "SopranoTextModel": "soprano",
     "Spark2_5ForCausalLM": "spark2_5",
     "SolarOpenForCausalLM": "glm",
     "StableLMEpochForCausalLM": "stablelm",
@@ -334,7 +315,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
 
 MMPROJ_MODEL_MAP: dict[str, str] = {
     "AudioFlamingo3ForConditionalGeneration": "ultravox",
-    "ChatterboxModel": "chatterbox",
     "ClefModel": "clef",
     "CogVLMForCausalLM": "cogvlm",
     "Cohere2VisionForConditionalGeneration": "command_r",
@@ -342,14 +322,12 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "VllmD3Model": "vllm_d3",
     "DeepseekOCR2ForCausalLM": "deepseek",
     "DeepseekOCRForCausalLM": "deepseek",
-    "DeepseekV41ForCausalLM": "deepseek41",
     "DeepseekV4ForCausalLM": "deepseek",
     "Dots3NoteForCausalLM": "dots3",
     "Dots3NoteForConditionalGeneration": "dots3",
     "DotsOCRForCausalLM": "dotsocr",
     "EmbeddingGemma2Model": "gemma",
     "Exaone4_5_ForConditionalGeneration": "exaone",
-    "FishOmniModel": "drama3",
     "Gemma3ForConditionalGeneration": "gemma",
     "Gemma3nForConditionalGeneration": "gemma",
     "Gemma4ForConditionalGeneration": "gemma",
@@ -373,9 +351,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Lfm2AudioForConditionalGeneration": "lfm2",
     "D1OmniModel": "lfm2",
     "Lfm2VlForConditionalGeneration": "lfm2",
-    "Lfm3AudioForConditionalGeneration": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
     "LightOnOCRForConditionalGeneration": "lighton_ocr",
     "Llama4ForConditionalGeneration": "llama4",
     "LlavaForConditionalGeneration": "llava",
@@ -384,18 +359,14 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "MiniMaxM3SparseForConditionalGeneration": "minimax",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
     "MiniCPMV4_7ForConditionalGeneration": "minicpm",
-    "MiniCPMOCRForSequenceClassification": "qwen3vl",
-    "MiniCPMOcrForConditionalGeneration": "minicpm",
     "Mistral3ForConditionalGeneration": "llava",
     "Mistral3Model": "llava",
-    "Nemotron3_5TranscribeForConditionalGeneration": "nemotron",
     "NemotronH_Omni_Reasoning_V3": "nemotron",
     "NemotronH_Nano_VL_V2": "nemotron",
     "MuseGlimmerForConditionalGeneration": "muse_glimmer",
     "MuseSparkForConditionalGeneration": "muse_spark",
     "PaddleOCRVisionModel": "ernie",
     "Phi4ForCausalLMV": "phi",
-    "NEOVisionModel": "sensenova_u1",
     "Qwen2AudioForConditionalGeneration": "ultravox",
     "Qwen2VLForConditionalGeneration": "qwenvl",
     "Qwen2VLModel": "qwenvl",
@@ -404,7 +375,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Qwen3ASRForConditionalGeneration": "qwen3vl",
     "Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
     "PocketTTSModel": "pockettts",
-    "SopranoModel": "soprano",
     "Qwen3TTSForConditionalGeneration": "qwen3tts",
     "Qwen3VLForConditionalGeneration": "qwen3vl",
     "Qwen3VLMoeForConditionalGeneration": "qwen3vl",

conversion/base.py

@@ -16,7 +16,6 @@ from pathlib import Path
 from hashlib import sha256
 from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Literal, Sequence, TypeVar, cast
 from itertools import chain
-from math import prod
 from transformers import AutoConfig
 
 import numpy as np
@@ -156,7 +155,6 @@ class ModelBase:
         self.fuse_gate_up_exps = fuse_gate_up_exps
         self._gate_exp_buffer: dict[int, Tensor] = {}
         self._up_exp_buffer: dict[int, Tensor] = {}
-        self._unfusable_gate_up_layers: set[int] = set()
         self.fuse_qkv = fuse_qkv
         self._q_buffer: dict[int, Tensor] = {}
         self._k_buffer: dict[int, Tensor] = {}
@@ -167,8 +165,6 @@ class ModelBase:
         self._fusable_qkv_weight_layers: set[int] = set()
         self._fusable_qkv_bias_layers: set[int] = set()
         self.hparams = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) if hparams is None else hparams
-        # source tensor name to filtered tensor name mapping.
-        self._source_tensor_names: dict[str, str | None] = {}
         self.model_tensors = self.index_tensors(remote_hf_model_id=remote_hf_model_id)
         self.metadata_override = metadata_override
         self.model_name = model_name
@@ -179,9 +175,7 @@ class ModelBase:
         self._nvfp4_layer_algo: dict[str, str | None] = {} # per-layer quant_algo, keyed by HF module path
         self._prec_a4: dict[str, bool] = {} # gguf tensor name -> can use 4-bit (A4) activations
         self._fp8_as_q8 = fp8_as_q8
-        self._preserve_fp8 = ftype == gguf.LlamaFileType.GUESSED and not fp8_as_q8
         self._fp8_dequantized: set[str] = set()
-        self._fp8_e4m3_preserved: set[str] = set()
 
         # Apply heuristics to figure out typical tensor encoding based on first tensor's dtype
         # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.
@@ -227,25 +221,21 @@ class ModelBase:
         tensors: dict[str, Callable[[], Tensor]] = {}
 
         if remote_hf_model_id is not None:
+            is_safetensors = True
+
             logger.info(f"Using remote model with HuggingFace id: {remote_hf_model_id}")
             remote_tensors = gguf.utility.SafetensorRemote.get_list_tensors_hf_model(remote_hf_model_id)
             for name, remote_tensor in remote_tensors.items():
                 data_gen = lambda r=remote_tensor: LazyTorchTensor.from_remote_tensor(r)  # noqa: E731
-                titem = self.filter_tensors((name, data_gen))
-                self._source_tensor_names[name] = titem[0] if titem else None
-                if titem:
+                if titem := self.filter_tensors((name, data_gen)):
                     tname, tgen = titem
                     tensors[tname] = tgen
 
             return tensors
 
-        return self.index_local_tensors("model" if not self.is_mistral_format else "consolidated")
-
-    # index the local "<prefix>*.safetensors" files (or pytorch_model*.bin), through filter_tensors
-    def index_local_tensors(self, prefix: str) -> dict[str, Callable[[], Tensor]]:
-        tensors: dict[str, Callable[[], Tensor]] = {}
+        prefix = "model" if not self.is_mistral_format else "consolidated"
         part_names: list[str] = ModelBase.get_model_part_names(self.dir_model, prefix, ".safetensors")
-        is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / f"{prefix}.safetensors.index.json").is_file())
+        is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / "model.safetensors.index.json").is_file())
         if not is_safetensors:
             part_names = ModelBase.get_model_part_names(self.dir_model, "pytorch_model", ".bin")
 
@@ -253,7 +243,7 @@ class ModelBase:
         tensor_names_from_parts: set[str] = set()
 
         if not self.is_mistral_format:
-            index_name = f"{prefix}.safetensors" if is_safetensors else "pytorch_model.bin"
+            index_name = "model.safetensors" if is_safetensors else "pytorch_model.bin"
             index_name += ".index.json"
             index_file = self.dir_model / index_name
 
@@ -298,9 +288,7 @@ class ModelBase:
                             data_gen = lambda data=data_torch: LazyTorchTensor.from_eager(data)  # noqa: E731
                         else:
                             data_gen = lambda data=data_torch: data  # noqa: E731
-                    titem = self.filter_tensors((name, data_gen))
-                    self._source_tensor_names[name] = titem[0] if titem else None
-                    if titem:
+                    if titem := self.filter_tensors((name, data_gen)):
                         tname, tgen = titem
                         tensors[tname] = tgen
 
@@ -338,7 +326,7 @@ class ModelBase:
 
     def dequant_model(self):
         # If all quantized tensors were already handled (e.g. pure NVFP4), skip
-        if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv", ".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")) for k in self.model_tensors):
+        if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv")) for k in self.model_tensors):
             return
 
         tensors_to_remove: list[str] = []
@@ -362,12 +350,6 @@ class ModelBase:
             def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor:
                 scale = scale.float()
 
-                # ModelOpt block scales are pre-shaped for broadcasting, e.g. [out/128, 1, in/128, 1]
-                if block_size is None and scale.ndim == 2 * weight.ndim and all(n == 1 for n in scale.shape[1::2]):
-                    assert all(w % n == 0 for w, n in zip(weight.shape, scale.shape[0::2]))
-                    block_size = [w // n for w, n in zip(weight.shape, scale.shape[0::2])]
-                    scale = scale.reshape(tuple(scale.shape[0::2]))
-
                 if block_size is not None:
                     dim_offset = scale.ndim - len(block_size)
                     for i, size in enumerate(block_size):
@@ -458,7 +440,7 @@ class ModelBase:
 
                 return (unpacked * scale.unsqueeze(-1).float()).reshape(shape)
 
-            def dequant_fp8(fp8_configs: list[dict[str, Any]] | None = None) -> None:
+            def dequant_fp8() -> None:
                 for name in self.model_tensors.keys():
                     if name.endswith(".weight_scale"):
                         weight_name = name.removesuffix("_scale")
@@ -467,29 +449,14 @@ class ModelBase:
                             continue
                         w = self.model_tensors[weight_name]
                         s = self.model_tensors[name]
-                        block_size = None
                         is_fp8_weight = False
-                        if self._fp8_as_q8 or fp8_configs is not None:
-                            weight = w()
-                            is_fp8_weight = weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
-                        if fp8_configs is not None:
-                            if not is_fp8_weight:
-                                raise ValueError(f"Expected remaining weight {weight_name!r} to be FP8, got {weight.dtype}")
-                            scale = s()
-                            is_channel_scale = scale.ndim >= 1 and scale.ndim <= weight.ndim and scale.shape[0] == weight.shape[0] and all(size == 1 for size in scale.shape[1:])
-                            if prod(scale.shape) != 1 and not is_channel_scale:
-                                block_configs = [c for c in fp8_configs if c.get("strategy") == "block"]
-                                if not block_configs or any(not c.get("block_structure") or c.get("group_size") is not None for c in block_configs):
-                                    raise ValueError(f"FP8 weight {weight_name!r} with scale shape {list(scale.shape)} requires a block layout")
-                                block_sizes = {tuple(c["block_structure"]) for c in block_configs}
-                                if len(block_sizes) != 1:
-                                    raise ValueError(f"FP8 weight {weight_name!r} has ambiguous block layouts")
-                                block_size = next(iter(block_sizes))
-                        self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+                        if self._fp8_as_q8:
+                            is_fp8_weight = w().dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
+                        self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), None)
                         tensors_to_remove.append(name)
-                        if self._fp8_as_q8 and is_fp8_weight:
+                        if is_fp8_weight:
                             self._fp8_dequantized.add(weight_name)
-                    if name.endswith((".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")):
+                    if name.endswith((".input_scale", ".k_scale", ".v_scale")):
                         tensors_to_remove.append(name)
 
             if quant_method == "bitnet":
@@ -500,7 +467,7 @@ class ModelBase:
                         s = self.model_tensors[name]
                         self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s())
                         tensors_to_remove.append(name)
-            elif quant_method in ("fp8", "bsgemm-moe"):  # bsgemm-moe: drama-3, same block layout
+            elif quant_method == "fp8":
                 block_size = quant_config.get("weight_block_size")
                 for name in self.model_tensors.keys():
                     if name.endswith("_scale_inv"):
@@ -554,37 +521,29 @@ class ModelBase:
                 nvfp4_compressed_tensors = self._is_nvfp4_compressed_tensors(quant_method, quant_format, groups)
 
                 if nvfp4_compressed_tensors:
-                    fp8_configs = [
-                        g["weights"] for g in groups.values()
-                        if isinstance(g, dict) and g.get("format") == "float-quantized"
-                        and g.get("weights", {}).get("type") == "float"
-                        and g.get("weights", {}).get("num_bits") == 8
-                    ]
-                    dequant_fp8(fp8_configs)
+                    dequant_fp8()
                 elif len(groups) > 1:
                     raise NotImplementedError("Can't handle multiple config groups for compressed-tensors yet")
                 elif quant_format == "float-quantized" or quant_format == "int-quantized" or quant_format == "naive-quantized":
                     weight_config = tuple(groups.values())[0]["weights"]
                     block_size = weight_config.get("block_structure", None)
                     strategy = weight_config.get("strategy")
+                    assert strategy == "channel" or strategy == "block"
+                    assert weight_config.get("group_size") is None  # didn't find a model using this yet
                     is_fp8 = (
                         quant_format == "float-quantized"
                         and weight_config.get("type") == "float"
                         and weight_config.get("num_bits") == 8
                     )
-                    assert strategy in ("channel", "block") or (is_fp8 and strategy == "tensor")
-                    assert weight_config.get("group_size") is None  # didn't find a model using this yet
                     for name in self.model_tensors.keys():
                         if name.endswith(".weight_scale"):
                             weight_name = name.removesuffix("_scale")
                             w = self.model_tensors[weight_name]
                             s = self.model_tensors[name]
-                            self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+                            self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), block_size)
                             tensors_to_remove.append(name)
                             if self._fp8_as_q8 and is_fp8:
                                 self._fp8_dequantized.add(weight_name)
-                        if is_fp8 and name.endswith((".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")):
-                            tensors_to_remove.append(name)
                 elif quant_format == "pack-quantized":
                     weight_config = tuple(groups.values())[0]["weights"]
                     assert weight_config.get("strategy") == "group"
@@ -622,148 +581,6 @@ class ModelBase:
         for name, value in new_tensors.items():
             self.model_tensors[name] = value
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        new_name = self.tensor_map.get_name(name, try_suffixes=(".weight",))
-        return (new_name,) if new_name is not None else ()
-
-    def _prepare_fp8_e4m3_tensors(self):
-        if not self._preserve_fp8:
-            return
-
-        scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
-        input_scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
-        consumed: list[str] = []
-        candidates: dict[str, tuple[str, Tensor, tuple[str, ...], torch.Size]] = {}
-        weight_dtypes: dict[str, torch.dtype] = {}
-        scaled_weights: set[str] = set()
-        expert_pattern = re.compile(r"(\.[^.]*experts)\.(\d+)\.")
-        scale_types = {
-            gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V,
-            gguf.MODEL_TENSOR.ATTN_OUT, gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE,
-            gguf.MODEL_TENSOR.FFN_GATE, gguf.MODEL_TENSOR.FFN_UP, gguf.MODEL_TENSOR.FFN_DOWN,
-            gguf.MODEL_TENSOR.FFN_GATE_SHEXP, gguf.MODEL_TENSOR.FFN_UP_SHEXP, gguf.MODEL_TENSOR.FFN_DOWN_SHEXP,
-            gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
-            gguf.MODEL_TENSOR.SSM_IN, gguf.MODEL_TENSOR.SSM_OUT, gguf.MODEL_TENSOR.SSM_ALPHA, gguf.MODEL_TENSOR.SSM_BETA,
-            gguf.MODEL_TENSOR.NEXTN_EH_PROJ, gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, gguf.MODEL_TENSOR.OUTPUT,
-        }
-
-        for scale_name in list(self.model_tensors.keys()):
-            weight_name = None
-            if scale_name.endswith(".weight_scale"):
-                weight_name = scale_name.removesuffix("_scale")
-            elif scale_name.endswith("_scale_inv"):
-                weight_name = scale_name.removesuffix("_scale_inv")
-            elif scale_name.endswith(".qscale_weight"):
-                weight_name = scale_name.removesuffix("qscale_weight") + "weight"
-
-            if weight_name is None or weight_name not in self.model_tensors:
-                continue
-
-            scaled_weights.add(weight_name)
-            weight = self.model_tensors[weight_name]()
-            weight_dtypes[weight_name] = weight.dtype
-            if weight.dtype != torch.float8_e4m3fn or weight.ndim < 2:
-                continue
-
-            scale = self.model_tensors[scale_name]()
-            if prod(scale.shape) != 1:
-                continue
-
-            merged_name = expert_pattern.sub(r"\1.", weight_name)
-            new_names = self._map_fp8_weight_names(merged_name)
-            if not new_names:
-                continue
-            if any((mapped := self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",))) is None or mapped[0] not in scale_types for new_name in new_names):
-                continue
-            bid = next((int(part) for part in new_names[0].split(".") if part.isdecimal()), None)
-            self._fp8_e4m3_preserved.add(weight_name)
-            if any(new_name.endswith("_norm.weight") or self.tensor_force_quant(weight_name, new_name, bid, weight.ndim) != gguf.GGMLQuantizationType.F8_E4M3 for new_name in new_names):
-                self._fp8_e4m3_preserved.discard(weight_name)
-                continue
-            candidates[weight_name] = (scale_name, scale, new_names, weight.shape)
-
-        expert_groups: dict[str, list[str]] = {}
-        for name in self.model_tensors:
-            if (match := expert_pattern.search(name)) and name.endswith(".weight"):
-                expert_groups.setdefault(name[:match.end(1)], []).append(name)
-        for names in expert_groups.values():
-            for name in names:
-                if name not in weight_dtypes:
-                    weight_dtypes[name] = self.model_tensors[name]().dtype
-            dtypes = {weight_dtypes[name] for name in names}
-            if torch.float8_e4m3fn not in dtypes:
-                continue
-            if dtypes == {torch.float8_e4m3fn} and not any(name in scaled_weights for name in names):
-                continue
-            # The converter merges expert projections together.
-            if not all(name in candidates for name in names):
-                for name in names:
-                    candidates.pop(name, None)
-                    self._fp8_e4m3_preserved.discard(name)
-                    if name not in scaled_weights:
-                        gen = self.model_tensors[name]
-                        self.model_tensors[name] = lambda gen=gen: gen().float()
-
-        for weight_name, (scale_name, scale, new_names, shape) in candidates.items():
-            scale = LazyTorchTensor.to_eager(scale).float().flatten()
-
-            weight_prefix = weight_name.removesuffix(".weight")
-            # Transformers fine-grained FP8 uses activation_scale while ModelOpt uses input_scale.
-            # Ref: https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/mistral3.py#L357-L361
-            input_scale_name = next((name for name in (
-                weight_prefix + ".input_scale",
-                weight_prefix + ".activation_scale",
-                weight_prefix + "_activation_scale",
-                weight_prefix + ".qscale_act",
-            ) if name in self.model_tensors), None)
-            input_scale = None
-            if input_scale_name is not None:
-                input_scale = self.model_tensors[input_scale_name]()
-                if prod(input_scale.shape) != 1:
-                    raise ValueError(f"FP8 input scale {input_scale_name!r} must be a scalar")
-                input_scale = LazyTorchTensor.to_eager(input_scale).float().flatten()
-                consumed.append(input_scale_name)
-
-            consumed.append(scale_name)
-
-            expert_match = expert_pattern.search(weight_name)
-            if expert_match:
-                expert_id = int(expert_match.group(2))
-                for new_name in new_names:
-                    target_name = new_name.replace(".weight", ".scale")
-                    entries = scale_tensors.setdefault(target_name, [])
-                    assert isinstance(entries, list)
-                    cast(list[tuple[int, float]], entries).append((expert_id, float(scale[0])))
-                    if input_scale is not None:
-                        target_name = new_name.replace(".weight", ".input_scale")
-                        entries = input_scale_tensors.setdefault(target_name, [])
-                        assert isinstance(entries, list)
-                        cast(list[tuple[int, float]], entries).append((expert_id, float(input_scale[0])))
-            else:
-                for new_name in new_names:
-                    mapped = self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",)) if len(shape) == 3 else None
-                    is_packed_expert = mapped is not None and mapped[0] in (
-                        gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
-                    )
-                    # Packed expert weights need one scale value per expert.
-                    n_scales = shape[0] if is_packed_expert else 1
-                    scale_tensors[new_name.replace(".weight", ".scale")] = np.repeat(scale.numpy(), n_scales)
-                    if input_scale is not None:
-                        input_scale_tensors[new_name.replace(".weight", ".input_scale")] = np.repeat(input_scale.numpy(), n_scales)
-
-        for name in consumed:
-            self.model_tensors.pop(name, None)
-
-        for name, values in chain(scale_tensors.items(), input_scale_tensors.items()):
-            if isinstance(values, list):
-                values.sort(key=lambda item: item[0])
-                scale = np.array([item[1] for item in values], dtype=np.float32)
-            else:
-                scale = values.astype(np.float32)
-            if not np.allclose(scale, 1.0, atol=1e-6):
-                logger.info(f"  + {name} (FP8 scale, shape [{scale.size}])")
-                self.gguf_writer.add_tensor(name, scale)
-
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         name, gen = item
@@ -808,26 +625,6 @@ class ModelBase:
             raise ValueError(f"Can not map tensor {name!r}")
         return new_name
 
-    def prepare_gate_up_fusion(self) -> None:
-        self._unfusable_gate_up_layers.clear()
-        if not self.fuse_gate_up_exps or self._fp8_as_q8:
-            return
-
-        for name, gen in self.model_tensors.items():
-            merged_name = re.sub(r"(\.experts)\.\d+\.", r"\1.", name)
-            mapped = self.tensor_map.get_type_and_name(merged_name, try_suffixes=(".weight",))
-            if mapped is None:
-                continue
-            tensor_type, new_name = mapped
-            if tensor_type not in (gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_GATE_UP_EXP) or not new_name.endswith(".weight"):
-                continue
-            if gen().dtype == torch.float8_e4m3fn:
-                bid = next(int(part) for part in new_name.split(".") if part.isdecimal())
-                self._unfusable_gate_up_layers.add(bid)
-
-        for bid in sorted(self._unfusable_gate_up_layers):
-            logger.warning(f"Skipping --fuse-gate-up-exps for layer {bid}: fusion of preserved FP8 expert projections is not supported")
-
     def prepare_qkv_fusion(self) -> None:
         self._fusable_qkv_weight_layers.clear()
         self._fusable_qkv_bias_layers.clear()
@@ -841,7 +638,6 @@ class ModelBase:
         }
         weights: dict[int, set[gguf.MODEL_TENSOR]] = {}
         biases: dict[int, set[gguf.MODEL_TENSOR]] = {}
-        fp8_layers: set[int] = set()
 
         for name in self.model_tensors:
             mapped = self.tensor_map.get_type_and_name(name, try_suffixes=(".weight", ".bias"))
@@ -856,19 +652,12 @@ class ModelBase:
                 continue
             if new_name.endswith(".weight"):
                 weights.setdefault(bid, set()).add(tensor_type)
-                if not self._fp8_as_q8 and self.model_tensors[name]().dtype == torch.float8_e4m3fn:
-                    fp8_layers.add(bid)
             elif new_name.endswith(".bias"):
                 biases.setdefault(bid, set()).add(tensor_type)
 
         for bid, weight_types in weights.items():
             bias_types = biases.get(bid, set())
             if weight_types == qkv_types and (not bias_types or bias_types == qkv_types):
-                # NVFP4 weights use separate reordering and repacking hooks and are exported before this fusion pass.
-                # Separate FP8 scales cannot be represented by one fused QKV scale.
-                if bid in fp8_layers:
-                    logger.warning(f"Skipping --fuse-qkv for layer {bid}: fusion of preserved FP8 projections is not supported")
-                    continue
                 self._fusable_qkv_weight_layers.add(bid)
                 if bias_types:
                     self._fusable_qkv_bias_layers.add(bid)
@@ -1057,8 +846,7 @@ class ModelBase:
         new_name = self.map_tensor_name(name)
 
         # Handle gate/up expert tensor fusion if enabled
-        # Preserved FP8 gate/up scales must stay with their separate projections.
-        if self.fuse_gate_up_exps and bid is not None and bid not in self._unfusable_gate_up_layers:
+        if self.fuse_gate_up_exps and bid is not None:
             if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_GATE_EXP, bid):
                 self._gate_exp_buffer[bid] = data_torch
             elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_UP_EXP, bid):
@@ -1115,46 +903,8 @@ class ModelBase:
 
         return [(new_name, data_torch)]
 
-    def _tensor_requires_f32(self, new_name: str, bid: int | None, n_dims: int) -> bool:
-        # Conditions should closely match those in llama_model_quantize_internal in llama.cpp.
-        return n_dims <= 1 or new_name.endswith("_norm.weight") or (
-            any(
-                self.match_model_tensor_name(new_name, key, bid)
-                for key in (
-                    gguf.MODEL_TENSOR.FFN_GATE_INP,
-                    gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
-                    gguf.MODEL_TENSOR.POS_EMBD,
-                    gguf.MODEL_TENSOR.TOKEN_TYPES,
-                    gguf.MODEL_TENSOR.SSM_CONV1D,
-                    gguf.MODEL_TENSOR.SHORTCONV_CONV,
-                    gguf.MODEL_TENSOR.TIME_MIX_FIRST,
-                    gguf.MODEL_TENSOR.TIME_MIX_W1,
-                    gguf.MODEL_TENSOR.TIME_MIX_W2,
-                    gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,
-                    gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,
-                    gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,
-                    gguf.MODEL_TENSOR.POSNET_NORM1,
-                    gguf.MODEL_TENSOR.POSNET_NORM2,
-                    gguf.MODEL_TENSOR.V_ENC_EMBD_POS,
-                    gguf.MODEL_TENSOR.A_ENC_EMBD_POS,
-                    gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,
-                    gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,
-                    # Kimi KDA conv weights should be F32
-                    gguf.MODEL_TENSOR.SSM_CONV1D_Q,
-                    gguf.MODEL_TENSOR.SSM_CONV1D_K,
-                    gguf.MODEL_TENSOR.SSM_CONV1D_V,
-                    # DSA indexer weights should be F32
-                    gguf.MODEL_TENSOR.INDEXER_PROJ,
-                )
-            )
-            or new_name[-7:] not in (".weight", ".lora_a", ".lora_b")
-        )
-
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        if name in self._fp8_e4m3_preserved and n_dims >= 2:
-            if self._tensor_requires_f32(new_name, bid, n_dims):
-                return gguf.GGMLQuantizationType.F32
-            return gguf.GGMLQuantizationType.F8_E4M3
+        del new_name, bid  # unused
         # Force FP8-original tensors to Q8_0 when requested; Q8_0 is faster than F16/BF16.
         if self._fp8_as_q8 and name in self._fp8_dequantized and n_dims >= 2:
             return gguf.GGMLQuantizationType.Q8_0
@@ -1285,8 +1035,7 @@ class ModelBase:
         expert_scales: dict[tuple[int, str], list[tuple[int, float]]] = {}
         expert_input_scales: dict[tuple[int, str], list[tuple[int, float]]] = {}
         expert_shapes: dict[tuple[int, str], list[int]] = {}
-        expert_names: dict[tuple[int, str], str] = {}
-        n_experts = self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True) or 0
+        n_experts = self.find_hparam(["num_local_experts", "num_experts"], optional=True) or 0
         consumed: list[str] = []
 
         for name in self.model_tensors.keys():
@@ -1297,17 +1046,14 @@ class ModelBase:
             input_scale_name = name.replace(".weight", ".input_scale")
             if scale_name not in self.model_tensors:
                 continue
-            weight = self.model_tensors[name]()
-            scale = self.model_tensors[scale_name]()
+            # Force eager materialization of lazy tensors
+            weight = LazyTorchTensor.to_eager(self.model_tensors[name]())
+            scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]())
 
-            # Leave FP8 weights and their scales for _prepare_fp8_e4m3_tensors.
-            if weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2) or scale.ndim < 2:
+            # Skip non-NVFP4 tensors(e.g. 1D scale, or float8 weight)
+            if scale.ndim < 2 or weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
                 continue
-            if weight.ndim != 2 or scale.shape[0] != weight.shape[0] or scale.shape[1] * 8 != weight.shape[1]:
-                raise ValueError(f"NVFP4 weight {name!r} has incompatible shapes: weight {list(weight.shape)}, scale {list(scale.shape)}")
 
-            weight = LazyTorchTensor.to_eager(weight)
-            scale = LazyTorchTensor.to_eager(scale)
             scale2 = LazyTorchTensor.to_eager(self.model_tensors.get(scale2_name, lambda: torch.tensor(1.0))())
             input_scale = LazyTorchTensor.to_eager(self.model_tensors.get(input_scale_name, lambda: torch.tensor(1.0))())
 
@@ -1334,7 +1080,6 @@ class ModelBase:
                     expert_scales[key] = []
                     expert_input_scales[key] = []
                     expert_shapes[key] = shape
-                    expert_names[key] = name
                 expert_blocks[key].append((expert_id, raw.copy()))
                 # Collect per-expert scale2 (scalar per expert)
                 expert_scales[key].append((expert_id, float(scale2.float().sum())))
@@ -1343,13 +1088,13 @@ class ModelBase:
 
                 # Flush when all experts for this (layer, proj) are collected
                 if n_experts > 0 and len(expert_blocks[key]) >= n_experts:
-                    self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type)
+                    self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type)
             else:
                 self._repack_nvfp4(name, weight, scale, scale2, input_scale)
 
         # Flush any remaining experts (fallback if n_experts was unknown)
         for bid, proj_type in list(expert_blocks.keys()):
-            self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type)
+            self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type)
 
         # Remove consumed tensors so get_tensors/modify_tensors won't see them
         for name in consumed:
@@ -1360,7 +1105,7 @@ class ModelBase:
             if name.endswith((".k_scale", ".v_scale")):
                 del self.model_tensors[name]
 
-    def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type):
+    def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type):
         experts = expert_blocks.pop(key)
         scales = expert_scales.pop(key)
         input_scales = expert_input_scales.pop(key)
@@ -1372,7 +1117,7 @@ class ModelBase:
         new_name = self.map_tensor_name(merged_name)
         logger.info(f"Repacked {new_name} with shape [{len(experts)}, {shape[0]}, {shape[1]}] and quantization NVFP4")
         self.gguf_writer.add_tensor(new_name, merged, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
-        self._tag_prec_a4(expert_names.pop(key), new_name)
+        self._tag_prec_a4(merged_name, new_name)
 
         scales.sort(key=lambda x: x[0])
         self._write_scales_tensor(new_name.replace(".weight", ".scale"), [s[1] for s in scales])
@@ -1423,25 +1168,18 @@ class ModelBase:
         # Per-tensor NVFP4 precision.
         self._nvfp4_layer_algo = {}
         if quant_layers:
-            # quantized_layers keys are module paths in the source checkpoint, not tensor names
-            # resolve each source tensor to the closest quantized_layers parent and record the algo under its filtered name for _tag_prec_a4
-            layer_algo = {k: v.get("quant_algo") for k, v in quant_layers.items() if isinstance(v, dict)}
-            matched: set[str] = set()
-            for src_name, filtered_name in self._source_tensor_names.items():
-                module = src_name
-                while module := module.rpartition(".")[0]:
-                    if module in layer_algo:
-                        break
-                if not module:
+            # store all possible module paths and assert if a quantized layer is not in the model
+            modules: set[str] = set()
+            for name in self.model_tensors:
+                while name := name.rpartition(".")[0]:
+                    modules.add(name)
+
+            for layer_name, entry in quant_layers.items():
+                if not isinstance(entry, dict):
                     continue
-
-                matched.add(module)
-                # filter_tensors may drop a tensors so we need to check for that here.
-                if filtered_name is not None:
-                    self._nvfp4_layer_algo[filtered_name.removesuffix(".weight").removesuffix(".bias")] = layer_algo[module]
-
-            unmatched = layer_algo.keys() - matched
-            assert not unmatched, f"quantized_layers entries not in the model tensors: {sorted(unmatched)}"
+                if titem := self.filter_tensors((layer_name, lambda: torch.empty(0))):
+                    assert titem[0] in modules, f"quantized_layers entry {layer_name!r} is not in the model tensors"
+                    self._nvfp4_layer_algo[titem[0]] = entry.get("quant_algo")
 
         # NVFP4 weights are repacked and written directly to gguf_writer.
         # This must run before dequant_model so NVFP4 tensors are removed
@@ -1471,10 +1209,8 @@ class ModelBase:
                             self.model_tensors[input_scale_name] = inverse_scale(self.model_tensors.pop(name))
             self._generate_nvfp4_tensors()
 
-        self._prepare_fp8_e4m3_tensors()
         self.dequant_model()
 
-        self.prepare_gate_up_fusion()
         self.prepare_qkv_fusion()
 
         # Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
@@ -1488,21 +1224,10 @@ class ModelBase:
             if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
                 continue
 
-            # FP8 transforms must only reorder values; norms and vectors use numeric F32 transforms.
-            preserve_fp8_e4m3 = self._preserve_fp8 and data_torch.dtype == torch.float8_e4m3fn and data_torch.ndim >= 2 and not name.endswith(("norm.weight", "norm_f.weight"))
-            if preserve_fp8_e4m3:
-                for new_name in self._map_fp8_weight_names(name):
-                    mapped_bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None)
-                    if self._tensor_requires_f32(new_name, mapped_bid, data_torch.ndim):
-                        preserve_fp8_e4m3 = False
-                        break
-                if preserve_fp8_e4m3:
-                    self._fp8_e4m3_preserved.add(name)
-
             old_dtype = data_torch.dtype
 
             # convert any unsupported data types to float32
-            if data_torch.dtype not in (torch.float16, torch.float32) and name not in self._fp8_e4m3_preserved:
+            if data_torch.dtype not in (torch.float16, torch.float32):
                 data_torch = data_torch.to(torch.float32)
 
             # use the first number-like part of the tensor name as the block id
@@ -1513,14 +1238,51 @@ class ModelBase:
                     break
 
             for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):
-                n_dims = len(data_torch.shape)
+                # TODO: why do we squeeze here?
+                # data = data_torch.squeeze().numpy()
+                data = data_torch.numpy()
+
+                n_dims = len(data.shape)
                 data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)
 
                 # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors
                 if n_dims <= 1 or new_name.endswith("_norm.weight"):
                     data_qtype = gguf.GGMLQuantizationType.F32
 
-                if data_qtype is False and self._tensor_requires_f32(new_name, bid, n_dims):
+                # Conditions should closely match those in llama_model_quantize_internal in llama.cpp
+                # Some tensor types are always in float32
+                if data_qtype is False and (
+                    any(
+                        self.match_model_tensor_name(new_name, key, bid)
+                        for key in (
+                            gguf.MODEL_TENSOR.FFN_GATE_INP,
+                            gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
+                            gguf.MODEL_TENSOR.POS_EMBD,
+                            gguf.MODEL_TENSOR.TOKEN_TYPES,
+                            gguf.MODEL_TENSOR.SSM_CONV1D,
+                            gguf.MODEL_TENSOR.SHORTCONV_CONV,
+                            gguf.MODEL_TENSOR.TIME_MIX_FIRST,
+                            gguf.MODEL_TENSOR.TIME_MIX_W1,
+                            gguf.MODEL_TENSOR.TIME_MIX_W2,
+                            gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,
+                            gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,
+                            gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,
+                            gguf.MODEL_TENSOR.POSNET_NORM1,
+                            gguf.MODEL_TENSOR.POSNET_NORM2,
+                            gguf.MODEL_TENSOR.V_ENC_EMBD_POS,
+                            gguf.MODEL_TENSOR.A_ENC_EMBD_POS,
+                            gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,
+                            gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,
+                            # Kimi KDA conv weights should be F32
+                            gguf.MODEL_TENSOR.SSM_CONV1D_Q,
+                            gguf.MODEL_TENSOR.SSM_CONV1D_K,
+                            gguf.MODEL_TENSOR.SSM_CONV1D_V,
+                            # DSA indexer weights should be F32
+                            gguf.MODEL_TENSOR.INDEXER_PROJ,
+                        )
+                    )
+                    or new_name[-7:] not in (".weight", ".lora_a", ".lora_b")
+                ):
                     data_qtype = gguf.GGMLQuantizationType.F32
 
                 if data_qtype is False and any(
@@ -1558,22 +1320,16 @@ class ModelBase:
                     else:
                         raise ValueError(f"Unknown file type: {self.ftype.name}")
 
-                if data_torch.dtype == torch.float8_e4m3fn:
-                    data = data_torch.view(torch.uint8).numpy() if data_qtype == gguf.GGMLQuantizationType.F8_E4M3 else data_torch.float().numpy()
-                else:
-                    data = data_torch.numpy()
-
                 # a chunked tensor quantizes as one chunk at a time, while it is written
                 quantize = data.quantize if isinstance(data, gguf.LazyChunkedTensor) else (
                     lambda qtype, d=data: gguf.quants.quantize(d, qtype))
 
-                if not (data_qtype == gguf.GGMLQuantizationType.F8_E4M3 and data.dtype == np.uint8):
-                    try:
-                        data = quantize(data_qtype)
-                    except gguf.QuantError as e:
-                        logger.warning("%s, %s", e, "falling back to F16")
-                        data_qtype = gguf.GGMLQuantizationType.F16
-                        data = quantize(data_qtype)
+                try:
+                    data = quantize(data_qtype)
+                except gguf.QuantError as e:
+                    logger.warning("%s, %s", e, "falling back to F16")
+                    data_qtype = gguf.GGMLQuantizationType.F16
+                    data = quantize(data_qtype)
 
                 shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
 
@@ -1614,8 +1370,6 @@ class ModelBase:
                 self.ftype = gguf.LlamaFileType.MOSTLY_NVFP4
             elif self._is_mxfp4:
                 self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
-            elif self._fp8_e4m3_preserved:
-                self.ftype = gguf.LlamaFileType.MOSTLY_F8_E4M3
 
         # Generate parameter weight class (useful for leader boards) if not yet determined
         if self.metadata.size_label is None and total_params > 0:
@@ -1947,7 +1701,7 @@ class TextModel(ModelBase):
             self.gguf_writer.add_expert_group_used_count(n_group_used)
             logger.info(f"gguf: expert groups used count = {n_group_used}")
 
-        if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func", "router_score_function"], optional=True)) is not None:
+        if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn", "router_score_func"], optional=True)) is not None:
             if score_func == "sigmoid":
                 self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
             elif score_func == "softmax":
@@ -2083,9 +1837,6 @@ class TextModel(ModelBase):
         if chkhsh == "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35":
             # ref: https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0
             res = "minerva-7b"
-        if chkhsh == "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262":
-            # ref: https://huggingface.co/domyn/Domyn-Small-v1.0
-            res = "domyn-small"
         if chkhsh == "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664":
             # ref: https://huggingface.co/tencent/Hunyuan-A13B-Instruct
             res = "hunyuan"
@@ -2131,9 +1882,6 @@ class TextModel(ModelBase):
         if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4":
             # ref: https://huggingface.co/evilfreelancer/ruGPT3XL
             res = "gpt-2"
-        if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9":
-            # ref: https://huggingface.co/Lernex/Metis-1.6-Logos
-            res = "gpt-2"
         if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
             # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
             res = "lfm2"
@@ -2191,9 +1939,6 @@ class TextModel(ModelBase):
         if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e":
             # ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0
             res = "cohere2moe"
-        if chkhsh == "d5420e601e979078c670bc503d9ac411c311fa76b76a76267c31e0d6ba816637":
-            # ref: https://huggingface.co/RecursiveSI/flint
-            res = "flint"
         if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":
             # ref: https://huggingface.co/Qwen/Qwen1.5-7B
             res = "qwen2"
@@ -2921,13 +2666,7 @@ class TextModel(ModelBase):
         )
 
         if vocab.tokenizer_type == MistralTokenizerType.tekken:
-            with open(vocab.fname_tokenizer, encoding="utf-8") as f:
-                pattern = json.load(f)["config"]["pattern"]
-            # newer tekken.json split numbers into right-aligned groups of 3 digits instead of single digits
-            if "\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))" in pattern:
-                self.gguf_writer.add_tokenizer_pre("tekken-num3")
-            else:
-                self.gguf_writer.add_tokenizer_pre("tekken")
+            self.gguf_writer.add_tokenizer_pre("tekken")
             self.gguf_writer.add_token_merges(
                 vocab.extract_vocab_merges_from_model()
             )
@@ -3404,7 +3143,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
     # Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
     # Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
     # latent MoE, situ, ...). Route on the top-level architecture instead.
-    if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
+    if model_type == ModelType.TEXT and arch in ("StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
         return arch
 
     # if "architectures" is found in the sub-config, use that instead

conversion/chatterbox.py

@@ -1,473 +0,0 @@
-# Chatterbox (ResembleAI) turbo and multilingual v3, from the raw checkpoints of the official repos
-# the checkpoints ship no config.json, the variant is detected from the talker file and the hparams come from the reference code
-# talker: text_emb and speech_emb are fused into one [text | speech] vocab, speech token i is <|speech_i|>
-# mmproj: the s3gen sidecar, the voice and conditioning encoders, the position and speech tables, and the conds.pt voice
-
-from __future__ import annotations
-
-import json
-import math
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Iterable
-
-import torch
-import torch.nn.functional as F
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger
-from .llama import LlamaModel
-
-# checkpoint file prefixes, the files are "<prefix>.safetensors"
-TURBO_TALKER = "t3_turbo_v1"
-MTL_TALKER   = "t3_mtl23ls_v3"
-TURBO_S3GEN  = "s3gen_meanflow"
-MTL_S3GEN    = "s3gen_v3"
-VE           = "ve"
-
-# relative speech ids shared by both variants (start/stop_speech_token in the reference)
-SPEECH_BOS = 6561
-SPEECH_EOS = 6562
-
-# sampling defaults of the reference, samplers it does not use are disabled so the common defaults do not apply
-TURBO_TOP_K = 1000
-MTL_TOP_K   = 0
-TURBO_SAMPLING = {"min_p": 0.0,  "top_p": 0.95, "temp": 0.8, "penalty_repeat": 1.2}
-MTL_SAMPLING   = {"min_p": 0.05, "top_p": 1.0,  "temp": 0.8, "penalty_repeat": 1.2}
-# the reference repetition penalty covers the whole generated history
-PENALTY_LAST_N = -1
-
-
-def _has_file(dir_model: Path, prefix: str) -> bool:
-    return (dir_model / f"{prefix}.safetensors").is_file()
-
-
-def _is_turbo(dir_model: Path) -> bool:
-    if _has_file(dir_model, TURBO_TALKER):
-        return True
-    if _has_file(dir_model, MTL_TALKER):
-        return False
-    raise FileNotFoundError(f"no chatterbox talker checkpoint in {dir_model}")
-
-
-# talker hparams of the reference (GPT2_MEDIUM_CONFIG / LLAMA_520M_CONFIG_DICT, T3Config)
-TURBO_TEXT_CONFIG = {
-    "n_layer": 24, "n_embd": 1024, "n_inner": 4096, "n_head": 16, "n_ctx": 8196, "layer_norm_epsilon": 1e-05,
-    "vocab_size": 50276, "speech_vocab_size": 6563,
-}
-MTL_TEXT_CONFIG = {
-    "num_hidden_layers": 30, "hidden_size": 1024, "intermediate_size": 4096, "num_attention_heads": 16,
-    "num_key_value_heads": 16, "head_dim": 64, "max_position_embeddings": 131072, "rms_norm_eps": 1e-05,
-    "rope_theta": 500000.0,
-    "rope_scaling": {"factor": 8.0, "high_freq_factor": 4.0, "low_freq_factor": 1.0, "original_max_position_embeddings": 8192},
-    "vocab_size": 2454, "speech_vocab_size": 8194,
-}
-
-
-def _is_chatterbox_checkpoint(dir_model: Path) -> bool:
-    return _has_file(dir_model, VE) and (_has_file(dir_model, TURBO_TALKER) or _has_file(dir_model, MTL_TALKER))
-
-
-@ModelBase.register_hparams_loader(_is_chatterbox_checkpoint)
-def _load_chatterbox_hparams(dir_model: Path) -> dict[str, Any]:
-    turbo = _is_turbo(dir_model)
-    logger.info(f"gguf: detected Chatterbox {'turbo' if turbo else 'multilingual v3'} checkpoint")
-    text_config = dict(TURBO_TEXT_CONFIG if turbo else MTL_TEXT_CONFIG)
-    # the mmproj tensors are named explicitly, they do not go through the block tensor map
-    return {"architectures": ["ChatterboxModel"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}}
-
-
-@ModelBase.register("ChatterboxModel")
-class ChatterboxTalkerModel(TextModel):
-    model_arch = gguf.MODEL_ARCH.LLAMA  # multilingual; the turbo constructor switches to GPT2
-
-    def __init__(self, dir_model: Path, *args, **kwargs):
-        self.is_turbo = _is_turbo(dir_model)
-        if self.is_turbo:
-            self.model_arch = gguf.MODEL_ARCH.GPT2
-        super().__init__(dir_model, *args, **kwargs)
-        self._text_embd: Tensor | None = None
-        self._speech_embd: Tensor | None = None
-        self._text_head: Tensor | None = None
-        self._speech_head: Tensor | None = None
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        return self.index_local_tensors(TURBO_TALKER if self.is_turbo else MTL_TALKER)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        # the reference feeds inputs_embeds, the transformer input embeddings are unused
-        if name in ("tfmr.wte.weight", "tfmr.embed_tokens.weight"):
-            return None
-        if name.startswith(("cond_enc.", "text_pos_emb.", "speech_pos_emb.")):
-            return None
-        return super().filter_tensors(item)
-
-    def set_vocab(self):
-        if self.is_turbo:
-            self._set_vocab_turbo()
-        else:
-            self._set_vocab_mtl()
-
-    def _speech_token_names(self, n_speech: int) -> list[str]:
-        return [f"<|speech_{i}|>" for i in range(n_speech)]
-
-    def _set_vocab_turbo(self):
-        # stock GPT-2 BPE from the checkpoint dir, extended with the speech tokens
-        tokens, toktypes, tokpre = self.get_vocab_base()
-        n_text = len(tokens)
-        n_speech = self.hparams["speech_vocab_size"]
-        speech = self._speech_token_names(n_speech)
-        tokens += speech
-        toktypes += [gguf.TokenType.CONTROL] * n_speech
-
-        with open(self.dir_model / "merges.txt", "r", encoding="utf-8") as f:
-            merges = [line.rstrip("\n") for line in f if line.strip() and not line.startswith("#version")]
-
-        self._write_vocab(tokens, toktypes, merges, tokpre, n_text)
-
-    def _write_vocab(self, tokens: list[str], toktypes: list[int], merges: list[str], tokpre: str, n_text: int):
-        self.gguf_writer.add_tokenizer_model("gpt2")
-        self.gguf_writer.add_tokenizer_pre(tokpre)
-        self.gguf_writer.add_token_list(tokens)
-        self.gguf_writer.add_token_types(toktypes)
-        self.gguf_writer.add_token_merges(merges)
-
-        self.gguf_writer.add_bos_token_id(n_text + SPEECH_BOS)
-        self.gguf_writer.add_eos_token_id(n_text + SPEECH_EOS)
-        self.gguf_writer.add_add_bos_token(False)
-        self.gguf_writer.add_add_eos_token(False)
-
-        # the reference samples the speech head only
-        self.gguf_writer.add_suppress_tokens(list(range(n_text)))
-
-    def _set_vocab_mtl(self):
-        # the reference BPE is char-level, it is re-encoded to byte-level with extra merges that rebuild each multi-byte char
-        # its lowercasing is folded into the embedding table, so the prompt needs no text preprocessing
-        with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f:
-            tok = json.load(f)
-
-        byte_map = gguf.vocab.bytes_to_unicode()
-
-        def enc(s: str) -> str:
-            return "".join(byte_map[b] for b in s.encode("utf-8"))
-
-        n_text = self.hparams["vocab_size"]
-        tokens: list[str] = [f"[unused_{i}]" for i in range(n_text)]
-        toktypes = [int(gguf.TokenType.UNUSED)] * n_text
-        char_merges: list[str] = []
-        for t, i in tok["model"]["vocab"].items():
-            if t == " ":
-                # the reference substitutes [SPACE] before encoding, the raw space token is never used
-                continue
-            tokens[i] = enc(t)
-            toktypes[i] = int(gguf.TokenType.NORMAL)
-            if len(t) == 1 and len(t.encode("utf-8")) > 1:
-                parts = [byte_map[b] for b in t.encode("utf-8")]
-                for k in range(1, len(parts)):
-                    char_merges.append("".join(parts[:k]) + " " + parts[k])
-        for entry in tok.get("added_tokens", []):
-            tokens[entry["id"]] = entry["content"]
-            toktypes[entry["id"]] = int(gguf.TokenType.CONTROL)
-
-        # raw spaces tokenize to [SPACE] directly
-        space_id = tok["model"]["vocab"]["[SPACE]"]
-        tokens[space_id] = enc(" ")
-        toktypes[space_id] = int(gguf.TokenType.NORMAL)
-
-        n_speech = self.hparams["speech_vocab_size"]
-        tokens += self._speech_token_names(n_speech)
-        toktypes += [int(gguf.TokenType.CONTROL)] * n_speech
-
-        # chars are atomic in the reference, they must form before any of its merges
-        merges = char_merges
-        for m in tok["model"].get("merges", []):
-            a, b = m if isinstance(m, list) else m.split(" ")
-            merges.append(enc(a) + " " + enc(b))
-
-        self._write_vocab(tokens, toktypes, merges, "default", n_text)
-
-    def _mtl_case_fold_pairs(self) -> list[tuple[int, int]]:
-        # (upper_id, lower_id) of the single-char vocab entries, for every script in the vocab
-        with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f:
-            vocab = json.load(f)["model"]["vocab"]
-        pairs: list[tuple[int, int]] = []
-        for t, i in vocab.items():
-            if len(t) != 1:
-                continue
-            low = t.lower()
-            if low != t and len(low) == 1 and low in vocab:
-                pairs.append((i, vocab[low]))
-        return pairs
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        sampling = TURBO_SAMPLING if self.is_turbo else MTL_SAMPLING
-        self.gguf_writer.add_sampling_top_k(TURBO_TOP_K if self.is_turbo else MTL_TOP_K)
-        self.gguf_writer.add_sampling_min_p(sampling["min_p"])
-        self.gguf_writer.add_sampling_top_p(sampling["top_p"])
-        self.gguf_writer.add_sampling_temp(sampling["temp"])
-        self.gguf_writer.add_sampling_penalty_repeat(sampling["penalty_repeat"])
-        self.gguf_writer.add_sampling_penalty_last_n(PENALTY_LAST_N)
-        if not self.is_turbo:
-            self.gguf_writer.add_classifier_free_guidance(True)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if self.is_turbo:
-            return
-        # llama3 rope scaling baked into the rope_freqs factors tensor
-        rp = self.hparams["rope_scaling"]
-        dim = self.hparams["head_dim"]
-        base = self.hparams["rope_theta"]
-        freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
-        factor = rp["factor"]
-        low_freq_wavelen = rp["original_max_position_embeddings"] / rp["low_freq_factor"]
-        high_freq_wavelen = rp["original_max_position_embeddings"] / rp["high_freq_factor"]
-        rope_factors = []
-        for freq in freqs:
-            wavelen = 2 * math.pi / freq
-            if wavelen < high_freq_wavelen:
-                rope_factors.append(1)
-            elif wavelen > low_freq_wavelen:
-                rope_factors.append(factor)
-            else:
-                smooth = (rp["original_max_position_embeddings"] / wavelen - rp["low_freq_factor"]) / (rp["high_freq_factor"] - rp["low_freq_factor"])
-                rope_factors.append(1 / ((1 - smooth) / factor + smooth))
-        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
-
-    def _maybe_emit_fused(self) -> Iterable[tuple[str, Tensor]]:
-        if self._text_embd is not None and self._speech_embd is not None:
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD),
-                   torch.cat([self._text_embd, self._speech_embd], dim=0))
-            self._text_embd = None
-            self._speech_embd = None
-        if self._text_head is not None and self._speech_head is not None:
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT),
-                   torch.cat([self._text_head, self._speech_head], dim=0))
-            self._text_head = None
-            self._speech_head = None
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # generate_extra_tensors output comes back through here with its final name
-        if name.startswith("rope_freqs"):
-            yield (name, data_torch)
-            return
-        # fused [text | speech] vocab: embeddings and output head
-        if name == "text_emb.weight":
-            if not self.is_turbo:
-                # the reference lowercases its input, the uppercase rows take the lowercase ones
-                fold = list(range(data_torch.shape[0]))
-                for upper, lower in self._mtl_case_fold_pairs():
-                    fold[upper] = lower
-                data_torch = data_torch[torch.tensor(fold)]
-            self._text_embd = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "speech_emb.weight":
-            self._speech_embd = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "text_head.weight":
-            self._text_head = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "speech_head.weight":
-            self._speech_head = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "speech_head.bias":
-            # the text head has no bias, its zone is suppressed at sampling anyway
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT, suffix=".bias"), F.pad(data_torch, (self.hparams["vocab_size"], 0)))
-            return
-
-        assert name.startswith("tfmr.")
-        name = name[len("tfmr."):]
-
-        if self.is_turbo:
-            # HF GPT-2 layout: Conv1D style weights are stored transposed
-            if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight")):
-                data_torch = data_torch.transpose(1, 0)
-            yield (self.map_tensor_name(name), data_torch)
-            return
-
-        if name.endswith("q_proj.weight"):
-            data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], None)
-        if name.endswith("k_proj.weight"):
-            data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], self.hparams["num_key_value_heads"])
-        yield (self.map_tensor_name("model." + name), data_torch)
-
-
-@ModelBase.register("ChatterboxModel")
-class ChatterboxMmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    def __init__(self, dir_model: Path, *args, **kwargs):
-        self.is_turbo = _is_turbo(dir_model)
-        super().__init__(dir_model, *args, **kwargs)
-        self._wnorm_g: dict[str, Tensor] = {}
-        self._wnorm_v: dict[str, Tensor] = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        return self.global_config.get("audio_config")
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        tensors: dict[str, Callable[[], Tensor]] = {}
-        for prefix in (TURBO_S3GEN if self.is_turbo else MTL_S3GEN, VE, TURBO_TALKER if self.is_turbo else MTL_TALKER):
-            tensors.update(self.index_local_tensors(prefix))
-        return tensors
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        # the s3gen, ve and talker checkpoints have disjoint top-level names
-        name, gen = item
-        new_name: str | None = None
-        # batchnorm bookkeeping (over the gguf name length cap) and the tokenizer dsp buffers are unused
-        if name.endswith("num_batches_tracked") or name in ("tokenizer.window", "tokenizer._mel_filters"):
-            return None
-        for src, dst in (
-            ("flow.encoder.",           "a.gen.fenc."),
-            ("flow.decoder.estimator.", "a.gen.est."),
-            ("mel2wav.",                "a.gen.hift."),
-            ("speaker_encoder.",        "a.spk."),
-            ("tokenizer.",              "a.s3tok."),
-            ("flow.",                   "a.gen.flow."),  # input_embedding, encoder_proj, spk_embed_affine_layer
-            ("lstm.",                   "a.ve.lstm."),
-            ("proj.",                   "a.ve.proj."),
-            ("cond_enc.",               "a.cenc."),
-        ):
-            if name.startswith(src):
-                new_name = dst + name[len(src):]
-                break
-        if name == "text_pos_emb.emb.weight":
-            new_name = "a.gen.t3.text_pos_emb"
-        elif name == "speech_pos_emb.emb.weight":
-            new_name = "a.gen.t3.speech_pos_emb"
-        elif name == "speech_emb.weight":
-            new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD] + ".weight"
-        if new_name is None:
-            return None
-        return super().filter_tensors((new_name, gen))
-
-    def set_gguf_parameters(self):
-        self.gguf_writer.add_file_type(self.ftype)
-
-        # speaker encoder, the front-end hparams are hard-coded in clip.cpp
-        self.gguf_writer.add_clip_has_audio_encoder(True)
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX_SPKENC)
-        self.gguf_writer.add_audio_projection_dim(80)
-        self.gguf_writer.add_audio_num_mel_bins(80)
-        self.gguf_writer.add_audio_block_count(0)
-        self.gguf_writer.add_audio_embedding_length(192)
-        self.gguf_writer.add_audio_head_count(1)
-        self.gguf_writer.add_audio_feed_forward_length(192)
-        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-
-        # audio generator (s3gen, chatterbox projector); the flow encoder shape
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX)
-        self.gguf_writer.add_gen_audio_projection_dim(80)
-        self.gguf_writer.add_gen_audio_embedding_length(512)
-        self.gguf_writer.add_gen_audio_feed_forward_length(2048)
-        self.gguf_writer.add_gen_audio_block_count(6)
-        self.gguf_writer.add_gen_audio_head_count(8)
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
-
-    def _fuse_weight_norm(self, name: str, data_torch: Tensor) -> tuple[str, Tensor] | None:
-        # torch weight_norm parametrization: weight = g * v / |v| over dims 1..n
-        base = name.split(".parametrizations.weight.original")[0]
-        if name.endswith("original0"):
-            self._wnorm_g[base] = data_torch
-        else:
-            self._wnorm_v[base] = data_torch
-        if base in self._wnorm_g and base in self._wnorm_v:
-            g = self._wnorm_g.pop(base)
-            v = self._wnorm_v.pop(base)
-            norm = v.float().norm(dim=tuple(range(1, v.dim())), keepdim=True)
-            return (base + ".weight", g.float() * v.float() / norm)
-        return None
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if ".parametrizations.weight.original" in name:
-            fused = self._fuse_weight_norm(name, data_torch)
-            if fused is not None:
-                yield fused
-            return
-        yield (name, data_torch)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        del name, bid
-        # tensors read on the host stay F32
-        if new_name.startswith(("a.ve.", "a.cenc.", "voice.", "a.gen.t3.", "a.gen.hift.m_source.")):
-            return gguf.GGMLQuantizationType.F32
-        # the conv ops have no BF16 kernels
-        if n_dims >= 2 and new_name.endswith((".weight", ".weight_v")):
-            return gguf.GGMLQuantizationType.F16
-        return gguf.GGMLQuantizationType.F32
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        conds = torch.load(self.dir_model / "conds.pt", map_location="cpu", weights_only=True)
-        t3c = conds["t3"] if isinstance(conds, dict) else conds.t3
-        genc = conds["gen"] if isinstance(conds, dict) else conds.gen
-        t3c = vars(t3c) if not isinstance(t3c, dict) else t3c
-        genc = vars(genc) if not isinstance(genc, dict) else genc
-
-        def talker_tensor(name: str) -> Tensor:
-            return LazyTorchTensor.to_eager(self.model_tensors[name]())
-
-        # the conds.pt voice: voice.default holds the talker conditioning rows, voice.default.ref_* the flow decoder reference
-        # 1D tensors are written as F32, the speech ids stay exact
-        yield ("voice.default.ref_tokens", genc["prompt_token"][0].float())
-        yield ("voice.default.ref_feat", genc["prompt_feat"][0].float())
-        yield ("voice.default.ref_spk", genc["embedding"][0].float())
-
-        spkr_w = talker_tensor("a.cenc.spkr_enc.weight").float()
-        spkr_b = talker_tensor("a.cenc.spkr_enc.bias").float()
-        spkr_row = spkr_w @ t3c["speaker_emb"][0].float() + spkr_b
-        speech_emb = talker_tensor(self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD)).float()
-        cond_tokens = t3c["cond_prompt_speech_tokens"][0]
-
-        if self.is_turbo:
-            # turbo talker conditioning: [spkr, speech rows of the cond tokens]
-            yield ("voice.default", torch.cat([spkr_row[None], speech_emb[cond_tokens]], dim=0))
-            return
-
-        # multilingual talker conditioning [spkr, perceiver x32, emotion], from the reference perceiver over the cond speech tokens
-        with torch.no_grad():
-            speech_pos = talker_tensor("a.gen.t3.speech_pos_emb").float()
-            pse = speech_emb[cond_tokens] + speech_pos[: cond_tokens.shape[0]]
-
-            ln_w = talker_tensor("a.cenc.perceiver.attn.norm.weight").float()
-            ln_b = talker_tensor("a.cenc.perceiver.attn.norm.bias").float()
-            wq = talker_tensor("a.cenc.perceiver.attn.to_q.weight").float()
-            bq = talker_tensor("a.cenc.perceiver.attn.to_q.bias").float()
-            wk = talker_tensor("a.cenc.perceiver.attn.to_k.weight").float()
-            bk = talker_tensor("a.cenc.perceiver.attn.to_k.bias").float()
-            wv = talker_tensor("a.cenc.perceiver.attn.to_v.weight").float()
-            bv = talker_tensor("a.cenc.perceiver.attn.to_v.bias").float()
-            wo = talker_tensor("a.cenc.perceiver.attn.proj_out.weight").float()
-            bo = talker_tensor("a.cenc.perceiver.attn.proj_out.bias").float()
-            query = talker_tensor("a.cenc.perceiver.pre_attention_query")[0].float()
-
-            n_head = 4
-            n_e = query.shape[1]
-
-            def attn_block(x1: Tensor, x2: Tensor) -> Tensor:
-                nx1 = F.layer_norm(x1, (n_e,), ln_w, ln_b)
-                nx2 = F.layer_norm(x2, (n_e,), ln_w, ln_b)
-                q = (nx1 @ wq.T + bq).view(-1, n_head, n_e // n_head).transpose(0, 1)
-                k = (nx2 @ wk.T + bk).view(-1, n_head, n_e // n_head).transpose(0, 1)
-                v = (nx2 @ wv.T + bv).view(-1, n_head, n_e // n_head).transpose(0, 1)
-                ctx = F.scaled_dot_product_attention(q, k, v)
-                ctx = ctx.transpose(0, 1).reshape(-1, n_e)
-                return ctx @ wo.T + bo + x1
-
-            pre = attn_block(query, pse)
-            p32 = attn_block(pre, pre)
-
-            emo = talker_tensor("a.cenc.emotion_adv_fc.weight").float()
-            emo_row = emo[:, 0] * t3c["emotion_adv"].reshape(-1)[0]
-            yield ("voice.default", torch.cat([spkr_row[None], p32, emo_row[None]], dim=0))

conversion/clover.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-
-from typing import Callable, Iterable, TYPE_CHECKING
-
-from .base import ModelBase, gguf
-from .qwen import _LinearAttentionVReorderBase
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-
-@ModelBase.register("CloverMoeModelLM")
-@ModelBase.example("Diffuspace/Clover-35B-A3B-Preview")
-class CloverMoeModel(_LinearAttentionVReorderBase):
-    model_arch = gguf.MODEL_ARCH.CLOVER_MOE
-    supports_mtp_export = False
-
-    def __init__(self, *args, **kwargs):
-        self.no_mtp = True
-        super().__init__(*args, **kwargs)
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self.gguf_writer.add_expert_shared_count(self.hparams.get("num_shared_experts", 1))
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        if name.endswith((".conv_right_weight", ".bd_softmax_logit_scale", ".bd_softmax_history_bias", ".bd_softmax_history_c")):
-            return None
-
-        return super().filter_tensors(item)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".A_log") or (name.endswith("norm.weight") and not name.endswith("linear_attn.norm.weight")):
-            data_torch = data_torch.float()
-
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/deepseek.py

@@ -236,12 +236,6 @@ class DeepseekModel(TextModel):
 class DeepseekV2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.DEEPSEEK2
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        # MLA projections do not have runtime weight-scale support.
-        if name.endswith((".q_a_proj.weight", ".q_b_proj.weight", ".kv_a_proj_with_mqa.weight", ".kv_b_proj.weight")):
-            return ()
-        return super()._map_fp8_weight_names(name)
-
     # TODO @ngxson : remove this when we support MTP for deepseek models
     skip_mtp = True
 
@@ -809,6 +803,9 @@ class DeepseekV4Model(TextModel):
         self._dsv4_mxfp4_generated = True
         return ()
 
+    def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str:
+        return self.format_tensor_name(key, bid, suffix)
+
     def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
         root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
             "embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"),
@@ -882,17 +879,17 @@ class DeepseekV4Model(TextModel):
 
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name):
-            return
+            return []
 
         # hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused
         if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]:
-            return
+            return []
 
         tensor_key, suffix = self._map_dsv4_tensor_name(name, bid)
         if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID:
-            return
+            return []
 
-        yield self.format_tensor_name(tensor_key, bid, suffix), data_torch
+        return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)]
 
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
         del bid  # unused

conversion/deepseek41.py

@@ -1,464 +0,0 @@
-from __future__ import annotations
-
-import json
-import math
-import re
-from pathlib import Path
-from typing import Any, Callable, Iterable, TYPE_CHECKING, cast
-
-import numpy as np
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import LazyTorchTensor, ModelBase, TextModel, gguf
-from .deepseek import DeepseekV4FlashVisionModel, DeepseekV4Model
-
-
-# Engram hash constants from inference/engram.py, stored in the GGUF: the token map needs the HF tokenizers
-# normalizers and the multipliers numpy's PCG64 stream, neither of which the runtime has.
-
-def _is_prime(n: int) -> bool:
-    if n < 2 or n % 2 == 0:
-        return n == 2
-    return all(n % d for d in range(3, math.isqrt(n) + 1, 2))
-
-
-def engram_primes(n_layers: int, max_ngram_size: int, n_heads: int, vocab_size: int) -> list[list[list[int]]]:
-    """EngramLayout.from_args: [layer][ngram order 2..max][head] bucket moduli, never reused across layers."""
-    seen: set[int] = set()
-    out: list[list[list[int]]] = []
-    for _ in range(n_layers):
-        per_ngram = []
-        for _ in range(max_ngram_size - 1):
-            sizes, cur = [], vocab_size - 1
-            for _ in range(n_heads):
-                cand = cur + 1
-                while not _is_prime(cand) or cand in seen:
-                    cand += 1
-                cur = cand
-                seen.add(cur)
-                sizes.append(cur)
-            per_ngram.append(sizes)
-        out.append(per_ngram)
-    return out
-
-
-def engram_multipliers(layer_ids: list[int], max_ngram_size: int, compressed_vocab_size: int) -> list[list[int]]:
-    """compute_hash_multipliers: odd, bounded so that token_id * multiplier cannot overflow int64."""
-    bound = max(1, (np.iinfo(np.int64).max // compressed_vocab_size) // 2)
-    rows = []
-    for layer_id in layer_ids:
-        rng = np.random.default_rng(10007 * layer_id)
-        values = rng.integers(low=0, high=bound, size=(max_ngram_size,), dtype=np.int64)
-        rows.append([int(v) * 2 + 1 for v in values])
-    return rows
-
-
-def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
-    """build_compressed_token_map: token id -> compressed id (tokens that normalize alike collapse)."""
-    from tokenizers import Regex, normalizers
-    from transformers import AutoTokenizer
-
-    tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
-    sentinel = "\ue000"
-    normalizer = normalizers.Sequence([
-        normalizers.NFKC(),
-        normalizers.NFD(),
-        normalizers.StripAccents(),
-        normalizers.Lowercase(),
-        normalizers.Replace(Regex(r"[ \t\r\n]+"), " "),
-        normalizers.Replace(Regex(r"^ $"), sentinel),
-        normalizers.Strip(),
-        normalizers.Replace(sentinel, " "),
-    ])
-    backend = tokenizer.backend_tokenizer
-    key_to_new: dict[str, int] = {}
-    lookup = [0] * len(tokenizer)
-    for token_id in range(len(tokenizer)):
-        text = backend.decode([token_id], skip_special_tokens=False)
-        if "\ufffd" in text:
-            key = backend.id_to_token(token_id)
-        else:
-            normalized = normalizer.normalize_str(text)
-            key = normalized if normalized else text
-        lookup[token_id] = key_to_new.setdefault(key, len(key_to_new))
-    return lookup, len(key_to_new)
-
-
-class _StreamedBytes:
-    """Raw tensor bytes produced chunk by chunk at write time (an Engram table is ~200 GB in BF16)."""
-    def __init__(self, shape: tuple[int, ...], chunks: Callable[[], Iterable[np.ndarray]]):
-        self.shape = shape
-        self.dtype = np.dtype(np.uint8)
-        self.nbytes = int(np.prod(shape))
-        self._chunks = chunks
-
-    def tofile(self, fout) -> None:
-        written = 0
-        for chunk in self._chunks():
-            chunk = np.ascontiguousarray(chunk, dtype=np.uint8)
-            chunk.tofile(fout)
-            written += chunk.nbytes
-        if written != self.nbytes:
-            raise ValueError(f"streamed {written} bytes, expected {self.nbytes}")
-
-    def byteswap(self, inplace: bool = False):
-        raise NotImplementedError("streamed tensors are written little-endian only")
-
-
-def _e8m0_check(scale_u8: np.ndarray, name: str) -> None:
-    # e4m3 (|x| in [2^-9, 448]) * 2^(e - 127) is exact in BF16 for e in [3, 246]
-    if scale_u8.size and (int(scale_u8.min()) < 3 or int(scale_u8.max()) > 246):
-        raise ValueError(f"{name}: E8M0 scale exponents outside [3, 246], the dequantized values would not be exact")
-
-
-@ModelBase.register("DeepseekV41ForCausalLM")
-class DeepseekV41Model(DeepseekV4Model):
-    """DeepSeek-V4.1 text model, lossless: FP8 (e4m3, 32x32 ue8m0 blocks) linears dequantized to BF16 (F32 with
-    --outtype f32), routed experts copied bit-exact as MXFP4, Engram tables (e4m3 rows, one e8m0 scale per 32) as the
-    exact BF16 lookup (F32 with --outtype f32), router and Engram gate F32. DSpark (mtp.*) is not converted here
-    (--dspark, DeepseekV41DSparkModel); the vision tower goes to an mmproj (--mmproj, DeepseekV41VisionModel).
-    The Engram tables of all layers are written as one per_layer_token_embd (rows concatenated in layer order) with
-    the PLE keys of qwen4exp: absolute head offsets and bucket counts, so the runtime can read it lazily."""
-    model_arch = gguf.MODEL_ARCH.DEEPSEEK41
-    supports_mtp_export = False
-
-    _ENGRAM_EMBED_RE = re.compile(r"layers\.(\d+)\.engram\.embed\.(weight|scale)$")
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.hparams.setdefault("num_hash_layers", 0)  # no hash-routed layers in V4.1
-        self._f32 = self.ftype == gguf.LlamaFileType.ALL_F32
-        qc = self.hparams.get("quantization_config") or {}
-        self._fp8_block = tuple(qc.get("weight_block_size") or (32, 32))
-        if qc.get("scale_fmt", "ue8m0") != "ue8m0":
-            raise ValueError(f"unsupported FP8 scale format {qc.get('scale_fmt')!r}")
-        if self.hparams.get("gate_temp", 1.0) != 1.0:
-            raise ValueError("gate_temp != 1 is not supported (the router logits would need dividing by it)")
-
-        # V4.1 does not use the DeepSeek-V4 template; a template shipped with the model is still added by set_vocab
-        self.gguf_writer.remove_key(gguf.Keys.Tokenizer.CHAT_TEMPLATE)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        # num_hidden_layers lives in text_config, which is merged only after indexing; MTP stages are skipped anyway
-        return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        hp = self.hparams
-        w = self.gguf_writer
-        w.add_attention_kv_source_layers(hp["kv_source_layer_ids"])
-        w.add_indexer_source_layers(hp["index_source_layer_ids"])
-        if hp.get("candidate_source_layer_id", -1) >= 0:
-            w.add_indexer_candidate_source_layer(hp["candidate_source_layer_id"])
-            w.add_indexer_candidate_top_k_blocks(hp["candidate_topk_blocks"])
-            w.add_indexer_candidate_block_size(hp["candidate_block_size"])
-
-        # GGUF cannot hold empty arrays: the Engram keys are omitted when the model has no Engram layers
-        layer_ids = list(hp.get("engram_layer_ids") or [])
-        if layer_ids:
-            max_ngram, n_heads = hp["engram_max_ngram_size"], hp["engram_n_heads"]
-            primes = engram_primes(len(layer_ids), max_ngram, n_heads, hp["engram_vocab_size"])
-            for li, layer_primes in enumerate(primes):
-                if sum(p for per in layer_primes for p in per) != hp["engram_num_embeddings"][li]:
-                    raise ValueError(f"engram layer {layer_ids[li]}: prime sum != num_embeddings")
-            token_map, cvocab = engram_compressed_token_map(self.dir_model)
-            if cvocab != hp["engram_compressed_vocab_size"] or len(token_map) != hp["vocab_size"]:
-                raise ValueError(f"compressed token map ({len(token_map)} -> {cvocab}) does not match the config")
-            # absolute row offsets into the concatenated table
-            sizes = [p for layer in primes for per in layer for p in per]
-            offsets = [sum(sizes[:i]) for i in range(len(sizes))]
-            w.add_key_value(gguf.Keys.PerLayerEmbedding.LAYERS.format(arch=w.arch), layer_ids,
-                            gguf.GGUFValueType.ARRAY, gguf.GGUFValueType.UINT32)
-            w.add_ple_ngram_size(max_ngram)
-            w.add_ple_heads_per_ngram(n_heads)
-            w.add_embedding_length_per_layer_input(hp["engram_head_dim"])
-            w.add_ple_layer_multipliers([m for row in engram_multipliers(layer_ids, max_ngram, cvocab) for m in row])
-            w.add_ple_head_offsets(offsets)
-            w.add_ple_head_vocab_sizes(sizes)
-            w.add_engram_pad_token_id(hp["engram_pad_token_id"])
-            w.add_engram_token_map(token_map)
-
-    def dequant_model(self):
-        bo, bi = self._fp8_block
-
-        def dequant(weight: Tensor, scale: Tensor) -> Tensor:
-            out_f, in_f = weight.shape
-            s = self._e8m0_to_float(scale).repeat_interleave(bo, 0)[:out_f].repeat_interleave(bi, 1)[:, :in_f]
-            return weight.float() * s
-
-        for name in [n for n in self.model_tensors if n.endswith(".scale") and not self._ENGRAM_EMBED_RE.match(n)]:
-            weight_name = name.removesuffix(".scale") + ".weight"
-            if weight_name not in self.model_tensors or self.model_tensors[weight_name]().dtype not in self._float8_dtypes():
-                continue
-            w, s = self.model_tensors[weight_name], self.model_tensors.pop(name)
-            out_f, in_f = w().shape
-            if tuple(s().shape) != (-(-out_f // bo), -(-in_f // bi)):
-                raise ValueError(f"{weight_name}: scale shape {tuple(s().shape)} does not match {(bo, bi)} blocks")
-            _e8m0_check(LazyTorchTensor.to_eager(s()).view(torch.uint8).numpy(), name)
-            self.model_tensors[weight_name] = lambda w=w, s=s: dequant(w(), s())
-            self._dsv4_fp8_dequantized.add(weight_name)
-
-    def _tensor_part(self, name: str) -> Path:
-        index = self.dir_model / "model.safetensors.index.json"
-        if not index.is_file():
-            return self.dir_model / "model.safetensors"
-        with open(index, "r", encoding="utf-8") as f:
-            return self.dir_model / json.load(f)["weight_map"][name]
-
-    def _write_mxfp4_experts(self, bid: int, proj: str, key: gguf.MODEL_TENSOR, n_expert: int | None = None) -> None:
-        n_expert = self.hparams["n_routed_experts"] if n_expert is None else n_expert
-        names = [f"layers.{bid}.ffn.experts.{e}.{proj}" for e in range(n_expert)]
-        pairs = [(self.model_tensors.pop(n + ".weight"), self.model_tensors.pop(n + ".scale")) for n in names]
-        rows, packed_cols = pairs[0][0]().shape
-
-        def chunks() -> Iterable[np.ndarray]:
-            for w, s in pairs:
-                yield self.repack_mxfp4_blocks(LazyTorchTensor.to_eager(w()), LazyTorchTensor.to_eager(s()))
-        data = cast(np.ndarray, _StreamedBytes((len(pairs), rows, packed_cols * 2 // 32 * 17), chunks))
-        self.gguf_writer.add_tensor(self.format_tensor_name(key, bid), data, raw_dtype=gguf.GGMLQuantizationType.MXFP4)
-
-    def _write_engram_tables(self, layer_ids: list[int]) -> None:
-        # official lookup e4m3 * 2^(e8m0 - 127), exact in BF16
-        parts, dim = [], None
-        for bid in layer_ids:
-            wn, sn = f"layers.{bid}.engram.embed.weight", f"layers.{bid}.engram.embed.scale"
-            rows, d = self.model_tensors.pop(wn)().shape
-            if d % 32 != 0 or tuple(self.model_tensors.pop(sn)().shape) != (rows, d // 32) or d != (dim or d):
-                raise ValueError(f"{wn}: expected one e8m0 scale per 32 elements and the same width in every layer")
-            if rows != self.hparams["engram_num_embeddings"][len(parts)]:
-                raise ValueError(f"{wn}: {rows} rows, config says {self.hparams['engram_num_embeddings'][len(parts)]}")
-            dim = d
-            parts.append((wn, sn, rows, self._tensor_part(wn), self._tensor_part(sn)))
-        assert dim is not None
-        name = self.format_tensor_name(gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD)
-        n_rows = sum(p[2] for p in parts)
-
-        def chunks() -> Iterable[np.ndarray]:
-            from safetensors import safe_open
-            for wn, sn, rows, wpart, spart in parts:
-                with safe_open(wpart, framework="pt") as fw, safe_open(spart, framework="pt") as fs:
-                    ws, ss = fw.get_slice(wn), fs.get_slice(sn)
-                    for r0 in range(0, rows, 1 << 18):
-                        q = ws[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 32)
-                        e = ss[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 1)
-                        _e8m0_check(e.numpy(), sn)
-                        v = (q.view(torch.float8_e4m3fn).float() * torch.exp2(e.float() - 127)).reshape(-1, dim)
-                        yield (v if self._f32 else v.bfloat16()).view(torch.uint8).numpy()
-        qt = gguf.GGMLQuantizationType.F32 if self._f32 else gguf.GGMLQuantizationType.BF16
-        data = cast(np.ndarray, _StreamedBytes((n_rows, dim * (4 if self._f32 else 2)), chunks))
-        self.gguf_writer.add_tensor(name, data, raw_dtype=qt)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if not self._dsv4_mxfp4_generated:
-            for bid in range(self.block_count):
-                self._write_mxfp4_experts(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)
-                self._write_mxfp4_experts(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)
-                self._write_mxfp4_experts(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)
-            if self.hparams.get("engram_layer_ids"):
-                self._write_engram_tables(list(self.hparams["engram_layer_ids"]))
-            self._dsv4_mxfp4_generated = True
-        return ()
-
-    _LAYER_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
-        "attn.indexer.wk.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_K, ".weight"),
-        "attn.indexer.k_norm.weight": (gguf.MODEL_TENSOR.INDEXER_K_NORM, ".weight"),
-        "engram.wkv.weight": (gguf.MODEL_TENSOR.ENGRAM_KV, ".weight"),
-    }
-
-    def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
-        m = re.match(r"layers\.\d+\.(.+)$", name)
-        if m is not None and m.group(1) in self._LAYER_MAP:
-            return self._LAYER_MAP[m.group(1)]
-        return super()._map_dsv4_tensor_name(name, bid)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".engram.k_weight"):
-            return  # folded into engram_gate
-        if name.endswith(".engram.q_weight"):
-            # Engram gate weight = q_weight * k_weight (bf16 * bf16 is exact in f32)
-            k = self.model_tensors[name.removesuffix("q_weight") + "k_weight"]
-            yield self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float()
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or name in self._dsv4_f32_tensors:
-            return gguf.GGMLQuantizationType.F32
-        if n_dims >= 2 and (name in self._dsv4_fp8_dequantized or name in self._dsv4_bf16_tensors):
-            return gguf.GGMLQuantizationType.F32 if self._f32 else gguf.GGMLQuantizationType.BF16
-        return False
-
-
-class DeepseekV41DSparkModel(DeepseekV41Model):
-    """DeepSeek-V4.1 DSpark speculative draft (mtp.0..2 of the checkpoint) as a DFLASH GGUF (`--dspark`).
-
-    Layout = the V4 DSpark draft (conversion/deepseek.py DeepseekV4DSparkModel) with V4.1's differences flagged by
-    `dflash.hyper_connection.single_pass`: stages blk.0..2 (MXFP4 experts bit-exact, dense BF16), mtp.0.main_proj /
-    main_norm -> fc / enc.output_norm, mtp.2.norm -> output_norm, markov_head.embed / head -> markov_w1 / markov_w2,
-    confidence_head.proj -> conf_proj (F32), no hc_head. Token embeddings and the lm head are the target's (shared at
-    runtime through ctx_other). target_layers are the V4.1 ids as-is: the draft reads the attention input of those
-    layers, which is what llama_set_embeddings_layer_inp extracts.
-    """
-    model_arch = gguf.MODEL_ARCH.DFLASH
-    mtp_only = True
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hp = self.hparams
-        self._n_main = hp["num_hidden_layers"]
-        self._n_stage = hp["num_nextn_predict_layers"]
-        if self._n_stage <= 0 or "dspark_block_size" not in hp:
-            raise ValueError("DeepSeek-V4.1 DSpark export needs num_nextn_predict_layers > 0 and the dspark_* config")
-        # stages keep main-model block ids (mtp.M -> layers.n_main+M); format_tensor_name() maps them back
-        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self._n_stage)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        hp = {**self.hparams, **self.hparams.get("text_config", {})}  # text_config is merged only after indexing
-        type(self)._dsv4_main_layers = hp["num_hidden_layers"]
-        type(self)._dsv4_nextn_layers = hp.get("num_nextn_predict_layers", 0)
-        return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        m = re.match(r"mtp\.(\d+)\.(.+)$", name)
-        if m is None:
-            return None  # shared with the target at runtime
-        stage, rest = int(m.group(1)), m.group(2)
-        assert cls._dsv4_main_layers is not None
-        if stage >= cls._dsv4_nextn_layers:
-            raise ValueError(f"Unexpected DSpark stage {stage}")
-        if rest.startswith(("markov_head.", "confidence_head.")):
-            return f"dspark.{rest}", gen
-        return f"layers.{cls._dsv4_main_layers + stage}.{rest}", gen
-
-    def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:
-        if bid is not None and bid >= self._n_main:
-            bid -= self._n_main
-        return super().format_tensor_name(key, bid, suffix)
-
-    def set_vocab(self):
-        super().set_vocab()
-        self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])
-
-    def set_gguf_parameters(self):
-        hp = self.hparams
-        n = self._n_stage
-        w = self.gguf_writer
-        w.add_block_count(n)
-        w.add_context_length(hp["max_position_embeddings"])
-        w.add_embedding_length(hp["hidden_size"])
-        w.add_feed_forward_length(hp["moe_intermediate_size"])
-        w.add_head_count(hp["num_attention_heads"])
-        w.add_head_count_kv(1)
-        w.add_key_length(hp["head_dim"])
-        w.add_value_length(hp["head_dim"])
-        w.add_rope_dimension_count(hp["qk_rope_head_dim"])
-        # compress-ratio-0 layers: plain rope at rope_theta, no YaRN
-        w.add_rope_freq_base(float(hp["rope_theta"]))
-        w.add_layer_norm_rms_eps(hp["rms_norm_eps"])
-        w.add_q_lora_rank(hp["q_lora_rank"])
-        w.add_sliding_window(hp["sliding_window"])
-        w.add_expert_count(hp["dspark_n_routed_experts"])
-        w.add_expert_used_count(hp["dspark_num_experts_per_tok"])
-        w.add_expert_feed_forward_length(hp["moe_intermediate_size"])
-        w.add_expert_shared_count(hp["n_shared_experts"])
-        w.add_expert_weights_scale(hp["routed_scaling_factor"])
-        w.add_expert_weights_norm(hp["norm_topk_prob"])
-        w.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS)
-        w.add_swiglu_clamp_exp([hp["swiglu_limit"]] * n)
-        w.add_swiglu_clamp_shexp([hp["swiglu_limit"]] * n)
-        w.add_attention_output_group_count(hp["o_groups"])
-        w.add_attention_output_lora_rank(hp["o_lora_rank"])
-        w.add_attention_compress_ratios([0] * n)
-        w.add_hyper_connection_count(hp["hc_mult"])
-        w.add_hyper_connection_sinkhorn_iterations(hp["hc_sinkhorn_iters"])
-        w.add_hyper_connection_epsilon(hp["hc_eps"])
-        w.add_hyper_connection_single_pass(True)
-        w.add_block_size(hp["dspark_block_size"])
-        w.add_target_layers(hp["dspark_target_layer_ids"])
-        w.add_sample_from_anchor(True)
-        w.add_has_confidence_head(True)
-        w.add_file_type(self.ftype)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # no Engram tables in the draft
-        if not self._dsv4_mxfp4_generated:
-            n_expert = self.hparams["dspark_n_routed_experts"]
-            for bid in range(self._n_main, self._n_main + self._n_stage):
-                self._write_mxfp4_experts(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP, n_expert)
-                self._write_mxfp4_experts(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP, n_expert)
-                self._write_mxfp4_experts(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP, n_expert)
-            self._dsv4_mxfp4_generated = True
-        return ()
-
-    _DSPARK_ROOT: dict[str, gguf.MODEL_TENSOR] = {
-        "main_proj.weight": gguf.MODEL_TENSOR.FC,
-        "main_norm.weight": gguf.MODEL_TENSOR.ENC_OUTPUT_NORM,
-        "norm.weight": gguf.MODEL_TENSOR.OUTPUT_NORM,
-        "dspark.markov_head.embed.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W1,
-        "dspark.markov_head.head.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W2,
-        "dspark.confidence_head.proj.weight": gguf.MODEL_TENSOR.DSPARK_CONF_PROJ,
-    }
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".ffn.gate.bias_vl"):
-            return
-        if name in self._DSPARK_ROOT:
-            yield self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch
-            return
-        m = re.match(r"layers\.\d+\.(main_proj\.weight|main_norm\.weight|norm\.weight)$", name)
-        if m is not None:
-            yield self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        if new_name == "conf_proj.weight":
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-
-@ModelBase.register("DeepseekV41ForCausalLM")
-@ModelBase.example("deepseek-ai/DeepSeek-V4.1-Flash-1017")
-class DeepseekV41VisionModel(DeepseekV4FlashVisionModel):
-    """DeepSeek-V4.1 vision tower (projector deepseek41v): the V4-Flash-Vision ViT + 3x3 aligner, with the V4.1 token
-    block ([START] ([IMAGE]*w [NEWLINE])*h [END], no pads) and resize plan. ref: inference/vision.py,
-    inference/image_processor.py of deepseek-ai/DeepSeek-V4.1-Flash-1017. The checkpoint config is HF-style
-    (vision_config); image_start / image_end / image_newline are written like V4's sentinels, there is no image_pad."""
-
-    def get_vision_config(self) -> dict[str, Any] | None:
-        vc = self.global_config.get("vision_config")
-        if not vc or vc.get("num_hidden_layers", 0) == 0:
-            raise ValueError("DeepseekV41VisionModel requires vision_config.num_hidden_layers > 0")
-        return {
-            "num_hidden_layers":   vc["num_hidden_layers"],
-            "hidden_size":         vc["hidden_size"],
-            "num_attention_heads": vc["num_attention_heads"],
-            "intermediate_size":   vc["intermediate_size"],
-            "patch_size":          vc["patch_size"],
-            # dynamic resolution; only used for compat / warmup
-            "image_size":          vc["patch_size"] * vc["downsample_ratio"] * 16,
-            "rope_theta":          vc.get("rope_theta", 10000.0),
-            "downsample_ratio":    vc["downsample_ratio"],
-            "min_pixels":          vc["min_pixels"],
-            "max_image_tokens":    vc["max_image_tokens"],
-            "max_wh_ratio":        vc.get("max_wh_ratio"),
-        }
-
-    def set_gguf_parameters(self):
-        assert self.hparams_vision is not None
-        super(DeepseekV4FlashVisionModel, self).set_gguf_parameters()
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK41V)
-        # pytorch default eps (vision.py), not the LLM rms_norm_eps (1e-20)
-        self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
-        self.gguf_writer.add_vision_use_silu(True)
-        self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
-        self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
-        # hardcoded in clip.cpp (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs
-        assert self.hparams_vision["max_image_tokens"] == 1024, self.hparams_vision["max_image_tokens"]
-        assert self.hparams_vision["max_wh_ratio"] is None, self.hparams_vision["max_wh_ratio"]
-        assert self.hparams_vision["rope_theta"] == 10000, self.hparams_vision["rope_theta"]

conversion/drama3.py

@@ -1,251 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, MmprojModel, gguf
-from .qwen import Qwen3_5MoeTextModel
-
-# Fish Audio Drama 3 (FishOmniModel)
-# text model: Qwen3.5-MoE backbone, samples the main token of each frame
-# mmproj: codebook tower and Fish DAC decoder (codec.pth)
-
-_TOWER_SHARED = "audio_decoder_mtp_shared.layers."
-_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
-
-
-@ModelBase.register("FishOmniModel")
-@ModelBase.example("fishaudio/drama-3")
-class Drama3TextModel(Qwen3_5MoeTextModel):
-    model_arch = gguf.MODEL_ARCH.QWEN35MOE
-    no_mtp = True  # the checkpoint has no MTP head
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        if item[0].startswith("audio_decoder"):
-            return None
-        return super().filter_tensors(item)
-
-
-@ModelBase.register("FishOmniModel")
-@ModelBase.example("fishaudio/drama-3")
-class Drama3MmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = False
-
-    _LAYER_MAP = {
-        "enorm.weight":                                 (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
-        "hnorm.weight":                                 (gguf.MODEL_TENSOR.A_GEN_CODE_HNORM, True),
-        "eh_proj.weight":                               (gguf.MODEL_TENSOR.A_GEN_CODE_EH_PROJ, False),
-        "semantic_embedding.weight":                    (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
-        "codebook_embedding.weight":                    (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
-        "final_layernorm.weight":                       (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM, True),
-        "transformer_layer.attention_norm.weight":      (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, True),
-        "transformer_layer.ffn_norm.weight":            (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, True),
-        "transformer_layer.attention.q_norm.weight":    (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, True),
-        "transformer_layer.attention.k_norm.weight":    (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, True),
-        "transformer_layer.attention.wo.weight":        (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, False),
-        "transformer_layer.feed_forward.w1.weight":     (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, False),
-        "transformer_layer.feed_forward.w3.weight":     (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, False),
-        "transformer_layer.feed_forward.w2.weight":     (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, False),
-    }
-
-    _heads: dict[int, Tensor]
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._heads = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        cfg = dict(self.global_config["audio_decoder_config"])
-        cfg["num_hidden_layers"] = cfg["shared_enrichment_layers"] + cfg["num_codebooks"] - 1
-        return cfg
-
-    def set_gguf_parameters(self):
-        cfg = self.hparams
-        self.gguf_writer.add_file_type(self.ftype)
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
-        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
-        self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
-        self.gguf_writer.add_gen_audio_block_count(cfg["num_hidden_layers"])
-        self.gguf_writer.add_gen_audio_head_count(cfg["n_head"])
-        self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
-        # other tower and codec hparams are hardcoded in clip.cpp
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name = item[0]
-        if name.startswith("audio_decoder") or name == "model.language_model.embed_tokens.weight":
-            return item
-        return None
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
-        if new_name.endswith(".weight") and (
-            new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
-            or (".up.blk." in new_name and new_name.endswith(".dwconv.weight"))
-            or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
-        ):
-            return gguf.GGMLQuantizationType.F16
-        # keep ConvTranspose1d kernels and folded codebook tables in F32
-        if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
-            return gguf.GGMLQuantizationType.F32
-        if new_name.startswith("a.gen.wav.quant."):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        T = gguf.MODEL_TENSOR
-        cfg = self.hparams
-
-        if name.startswith("a.gen.wav."):
-            yield (name, data_torch)
-            return
-
-        if name == "model.language_model.embed_tokens.weight":
-            # semantic band of the backbone embeddings
-            begin = self.global_config["semantic_start_token_id"]
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
-            return
-
-        if name == "audio_decoder.codebook_embeddings.weight":
-            n_cb = cfg["num_codebooks"]
-            yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(n_cb, -1, data_torch.shape[-1]))
-            return
-
-        if name.startswith("audio_decoder_mtp.heads."):
-            n_heads = cfg["num_codebooks"] - 1
-            self._heads[int(name.split(".")[2])] = data_torch
-            if len(self._heads) < n_heads:
-                return
-            yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), torch.stack([self._heads.pop(i) for i in range(n_heads)]))
-            return
-
-        for prefix, offset in ((_TOWER_SHARED, 0), (_TOWER_RESIDUAL, cfg["shared_enrichment_layers"])):
-            if not name.startswith(prefix):
-                continue
-            idx, key = name[len(prefix):].split(".", 1)
-            il = int(idx) + offset
-            if key == "transformer_layer.attention.wqkv.weight":
-                # rows are [query | gate | key | value]
-                n_q = cfg["n_head"] * cfg["head_dim"]
-                n_kv = cfg["n_local_heads"] * cfg["head_dim"]
-                q, gate, k, v = data_torch.split([n_q, n_q, n_kv, n_kv], dim=0)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, il), q)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_GATE, il), gate)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, il), k)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, il), v)
-                return
-            tensor, zero_centered = self._LAYER_MAP[key]
-            if zero_centered:
-                data_torch = data_torch.float() + 1
-            yield (self.format_tensor_name(tensor, il), data_torch)
-            return
-
-        raise ValueError(f"unexpected tensor {name}")
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # only the DAC decoder, the encoder is for voice cloning (not supported yet)
-        sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
-        sd = sd.get("state_dict", sd)
-        T = gguf.MODEL_TENSOR
-
-        def get(name: str) -> Tensor:
-            return sd[name].float()
-
-        def conv(prefix: str) -> Tensor:
-            # fold weight norm
-            if prefix + ".conv.weight" in sd:
-                return get(prefix + ".conv.weight")
-            g = get(prefix + ".conv.parametrizations.weight.original0")
-            v = get(prefix + ".conv.parametrizations.weight.original1")
-            return g * v / v.norm(dim=(1, 2), keepdim=True)
-
-        def quant_table(prefix: str) -> Tensor:
-            # fold out_proj into the codebook, so decoding is a row lookup
-            g = get(prefix + ".out_proj.weight_g")
-            v = get(prefix + ".out_proj.weight_v")
-            w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
-            return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
-
-        def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
-            alpha = alpha.flatten()
-            return alpha, 1.0 / (alpha + 1e-9)
-
-        q = "quantizer."
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
-        n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB),
-               torch.stack([quant_table(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]))
-
-        # post_module uses interleaved RoPE, permute q/k rows so NEOX RoPE gives the same result
-        tfm = q + "post_module."
-        n_head, d_head = 16, 64
-        order = torch.cat([torch.arange(0, d_head, 2), torch.arange(1, d_head, 2)])
-
-        def neox_rows(w: Tensor) -> Tensor:
-            return w.reshape(n_head, d_head, -1)[:, order].reshape(w.shape)
-
-        n_tfm = sum(1 for k in sd if k.startswith(tfm + "layers.") and k.endswith(".ffn_norm.weight"))
-        for il in range(n_tfm):
-            p = f"{tfm}layers.{il}."
-            wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_Q, il), neox_rows(wq))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_K, il), neox_rows(wk))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_V, il), wv)
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_OUT, il), get(p + "attention.wo.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_NORM, il), get(p + "attention_norm.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_SCALE, il), get(p + "attention_layer_scale.gamma"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_NORM, il), get(p + "ffn_norm.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_GATE, il), get(p + "feed_forward.w1.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_UP, il), get(p + "feed_forward.w3.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_DOWN, il), get(p + "feed_forward.w2.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_SCALE, il), get(p + "ffn_layer_scale.gamma"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get(tfm + "norm.weight"))
-
-        # upsample: 2x (ConvTranspose1d + ConvNeXt)
-        for il in range(2):
-            p = f"{q}upsample.{il}."
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".weight"), conv(p + "0"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".bias"), get(p + "0.conv.bias"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".weight"), conv(p + "1.dwconv"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".bias"), get(p + "1.dwconv.conv.bias"))
-            for key, tensor in (("norm", T.A_GEN_WAV_UP_NORM), ("pwconv1", T.A_GEN_WAV_UP_PW1), ("pwconv2", T.A_GEN_WAV_UP_PW2)):
-                yield (self.format_tensor_name(tensor, il, suffix=".weight"), get(f"{p}1.{key}.weight"))
-                yield (self.format_tensor_name(tensor, il, suffix=".bias"), get(f"{p}1.{key}.bias"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_GAMMA, il, suffix=""), get(p + "1.gamma"))
-
-        # DAC: entry conv, 4x (Snake, ConvTranspose1d, 3 residual units), Snake, post conv
-        d = "decoder.model."
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), conv(d + "0"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
-        for bid in range(4):
-            p = f"{d}{bid + 1}.block."
-            a, b = snake(get(p + "0.alpha"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(p + "1.conv.bias"))
-            for xid in range(3):
-                r = f"{p}{xid + 2}.block."
-                for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
-                    a, b = snake(get(r + act + ".alpha"))
-                    name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
-                    yield (name + ".alpha", a)
-                    yield (name + ".beta", b)
-                for cv, tensor in (("1", T.A_GEN_WAV_DAC_RES_CONV1), ("3", T.A_GEN_WAV_DAC_RES_CONV2)):
-                    name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
-                    yield (name + ".weight", conv(r + cv))
-                    yield (name + ".bias", get(r + cv + ".conv.bias"))
-        a, b = snake(get(d + "5.alpha"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias"))

conversion/flint.py

@@ -1,38 +0,0 @@
-from __future__ import annotations
-
-from .base import ModelBase, gguf
-from .qwen import Qwen2MoeModel
-
-
-@ModelBase.register("FlintForCausalLM")
-@ModelBase.example("RecursiveSI/flint")
-class FlintModel(Qwen2MoeModel):
-    model_arch = gguf.MODEL_ARCH.FLINT
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        dense = self.hparams["mlp_only_layers"]
-        if dense != list(range(len(dense))):
-            raise ValueError("Flint dense layers must precede the MoE layers")
-        self.gguf_writer.add_leading_dense_block_count(len(dense))
-        self.gguf_writer.add_expert_shared_count(
-            self.hparams["shared_expert_intermediate_size"] // self.hparams["moe_intermediate_size"]
-        )
-        self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
-        self.gguf_writer.add_expert_weights_scale(1.0)
-        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
-        self.gguf_writer.add_sliding_window_pattern([
-            kind == "sliding_attention" for kind in self.hparams["layer_types"]
-        ])
-        self.gguf_writer.add_rope_pattern([
-            i not in self.hparams["no_rope_layer_indices"] for i in range(self.block_count)
-        ])
-
-    def modify_tensors(self, data_torch, name, bid):
-        if name.endswith(".softmax_offset"):
-            name += ".weight"
-        if name.endswith(".gate.expert_bias"):
-            if not self.hparams["router_expert_bias"]:
-                return
-            name += ".bias"
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/lfm2.py

@@ -15,30 +15,25 @@ from .base import MmprojModel, ModelBase, TextModel, gguf, jinja_str_or_json, lo
 from .gemma import ConformerAudioModel
 
 
-def lfm2_feed_forward_length(hparams: dict[str, Any]) -> int:
-    ff_dim = hparams.get("block_ff_dim", hparams.get("intermediate_size"))
-    assert ff_dim is not None
-    auto_adjust_ff_dim = hparams["block_auto_adjust_ff_dim"]
-    ffn_dim_multiplier = hparams["block_ffn_dim_multiplier"]
-    multiple_of = hparams["block_multiple_of"]
-
-    if auto_adjust_ff_dim:
-        ff_dim = int(2 * ff_dim / 3)
-        # custom dim factor multiplier
-        if ffn_dim_multiplier is not None:
-            ff_dim = int(ffn_dim_multiplier * ff_dim)
-        ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
-
-    return ff_dim
-
-
 @ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
 @ModelBase.example("LiquidAI/LFM2-1.2B", "LiquidAI/LFM2.5-350M")
 class LFM2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.LFM2
 
     def _add_feed_forward_length(self):
-        self.gguf_writer.add_feed_forward_length(lfm2_feed_forward_length(self.hparams))
+        ff_dim = self.find_hparam(["block_ff_dim", "intermediate_size"])
+        auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]
+        ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]
+        multiple_of = self.hparams["block_multiple_of"]
+
+        if auto_adjust_ff_dim:
+            ff_dim = int(2 * ff_dim / 3)
+            # custom dim factor multiplier
+            if ffn_dim_multiplier is not None:
+                ff_dim = int(ffn_dim_multiplier * ff_dim)
+            ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
+
+        self.gguf_writer.add_feed_forward_length(ff_dim)
 
     def set_gguf_parameters(self):
         # set num_key_value_heads only for attention layers
@@ -77,9 +72,7 @@ def _is_d1_checkpoint(dir_model: Path) -> bool:
     if not (dir_model / "config.json").is_file():
         return False
     with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    # d1 on LFM3 is converted by conversion/lfm3.py
-    return config.get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") and not config.get("model_type", "").startswith("lfm3")
+        return json.load(f).get("auto_map", {}).get("AutoModel", "").endswith(".D1Model")
 
 
 @ModelBase.register_hparams_loader(_is_d1_checkpoint)

conversion/lfm3.py

@@ -1,629 +0,0 @@
-from __future__ import annotations
-
-import json
-import math
-
-from pathlib import Path
-from typing import Any, Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import MmprojModel, ModelBase, TextModel, gguf, logger
-from .lfm2 import D1Model, lfm2_feed_forward_length
-
-
-@ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration", "Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-1.7B", "LiquidAI/LFM3-1.7B-Base")
-class LFM3Model(TextModel):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # LFM3-Audio/Omni: encoder, projector and audio-output heads live next to the text model
-    _media_prefixes = ("encoder.", "mlp.", "codebook_embedding.", "depth_embeddings.", "depth_linear.", "depthformer.",
-                       "mask_embedding", "model.codebook_embedding.", "model.depth_decoder.")
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-
-        if name.startswith(cls._media_prefixes):
-            return None
-
-        name = name.removeprefix("lfm.")
-
-        return super().filter_tensors((name, gen))
-
-    # LFM3-Audio live ASR prompt of the model card, the audio follows in the assistant turn
-    _asr_template = (
-        "{{- bos_token -}}"
-        "{{- '<|im_start|>system\\nPerform live ASR.<|im_end|>\\n' -}}"
-        "{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}"
-    )
-
-    # TTS prompt of the model card: a language code is looked up, anything else is taken as the language name
-    _tts_langs = {
-        "ar": "Arabic", "da": "Danish", "de": "German", "el": "Greek", "en": "American English", "en-us": "American English",
-        "en-gb": "British English", "fi": "Finnish", "fr": "French", "he": "Hebrew", "it": "Italian", "ja": "Japanese",
-        "ko": "Korean", "nl": "Dutch", "pl": "Polish", "pt": "Portuguese", "es": "Spanish", "sv": "Swedish", "tr": "Turkish",
-        "zh": "Chinese",
-    }
-
-    # "language" and "voice" are optional, the text to speak is the first message
-    def _tts_template(self, think: bool) -> str:
-        return (
-            "{%- set names = " + json.dumps(self._tts_langs) + " -%}"
-            "{%- set code = (language | lower) if language is defined and language else 'en' -%}"
-            "{%- set lang = names[code] if code in names else language -%}"
-            "{%- set v = (voice | lower) if voice is defined and voice and (voice | lower) in ['female', 'male'] else 'female' -%}"
-            "{{- bos_token -}}"
-            "{{- '<|im_start|>system\\nPerform TTS in ' + lang + ' using a ' + v + ' voice.<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>user\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>assistant\\n" + ("<think></think>" if think else "") + "<|audio_start|>' -}}"
-        )
-
-    def set_vocab(self):
-        super().set_vocab()
-        # hparams hold the merged text config, the checkpoint's own model_type is in config.json
-        with open(self.dir_model / "config.json", encoding="utf-8") as f:
-            model_type = json.load(f).get("model_type")
-        is_audio = model_type == "lfm3_audio"
-        is_omni  = model_type == "lfm3_omni"
-        templates = []
-        if is_audio:
-            templates.append({"name": "asr", "template": self._asr_template})
-        if is_audio or is_omni:
-            # LFM3-Omni opens every assistant turn with an empty think block
-            templates.append({"name": "tts", "template": self._tts_template(think=is_omni)})
-        if templates:
-            self.gguf_writer.add_chat_template(templates)
-
-    def set_gguf_parameters(self):
-        layer_types = self.hparams["layer_types"]
-        n_kv_heads = self.hparams["num_key_value_heads"]
-
-        # set num_key_value_heads only for attention layers
-        self.hparams["num_key_value_heads"] = [
-            n_kv_heads if lt != "conv" else 0
-            for lt in layer_types
-        ]
-
-        super().set_gguf_parameters()
-
-        self.gguf_writer.add_rope_dimension_count(self.hparams["head_dim"])
-
-        self.gguf_writer.add_sliding_window_pattern([lt == "sliding_attention" for lt in layer_types])
-        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
-
-        self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # Partial RoPE is represented by masking trailing frequency pairs.
-        # rope_parameters is keyed by layer type; only full_attention layers use partial RoPE.
-        rope_parameters = self.hparams["rope_parameters"]["full_attention"]
-        partial = rope_parameters.get("partial_rotary_factor", 1.0)
-        head_dim = self.hparams["head_dim"]
-        rope_angles = int(partial * head_dim) // 2
-        n_pairs = head_dim // 2
-        assert 0 < rope_angles <= n_pairs
-        values = [1.0] * rope_angles + [1e30] * (n_pairs - rope_angles)
-        rope_freqs = torch.tensor(values, dtype=torch.float32)
-        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # conv op requires 2d tensor
-        if "conv.conv" in name:
-            data_torch = data_torch.squeeze(1)
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-# d1 on LFM3-VL / LFM3-Omni: the stock weights, the d1 prompt read at its last token
-_D1_TYPES = {"lfm3_vl": gguf.DecisionType.LFM3_D1, "lfm3_omni": gguf.DecisionType.LFM3_D1_OMNI}
-
-
-def _is_lfm3_d1_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("architectures") == ["D1Model"] and config.get("model_type") in _D1_TYPES
-
-
-@ModelBase.register_hparams_loader(_is_lfm3_d1_checkpoint)
-def _load_lfm3_d1_hparams(dir_model: Path) -> dict[str, Any]:
-    logger.info("gguf: detected LFM3 d1 checkpoint")
-    hparams = ModelBase.load_hparams(dir_model, False, guess=False)
-    hparams["d1_family"] = hparams["model_type"]
-    # the mmproj is the base model's, the text model writes the decision
-    hparams["architectures"] = ["Lfm3VlForConditionalGeneration" if hparams["model_type"] == "lfm3_vl" else "Lfm3OmniForConditionalGeneration"]
-    hparams["text_config"]["architectures"] = ["Lfm3D1Model"]
-    return hparams
-
-
-@ModelBase.register("Lfm3D1Model")
-@ModelBase.example("LiquidAI/d1-2B", "LiquidAI/d1-omni-2B")
-class LFM3D1Model(LFM3Model):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # how the checkpoint serves, hard-coded in the server: assert the checkpoint records the same
-    _served = {"d1_codes": 2, "d1_attention": "set"}
-
-    def set_vocab(self):
-        # the vocab without the speech task templates of LFM3-Audio/Omni, d1 only decides
-        TextModel.set_vocab(self)
-        # the d1 prompt of LFM2, the LFM3 template writes an empty thought before the answer
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": D1Model._systemone_template() + "<think></think>"}])
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        for key, value in self._served.items():
-            if self.hparams.get(key) != value:
-                raise ValueError(f"unsupported d1 checkpoint: {key} = {self.hparams.get(key)!r}, expected {value!r}")
-
-        # a lone question reads the whole prompt both ways on the full attention layers
-        self.gguf_writer.add_causal_attention(False)
-        self.gguf_writer.add_decision_type(_D1_TYPES[self.hparams["d1_family"]])
-
-        # the option answers share one temperature
-        with open(self.dir_model / "calibration.json", encoding="utf-8") as f:
-            calibration = json.load(f)
-        if calibration.get("temperatures"):
-            raise ValueError("unsupported d1 calibration: per-bucket temperatures")
-        for name in ("noul", "choice", "score"):
-            self.gguf_writer.add_decision_temperature(name, calibration["default"])
-
-
-class LFM3VLVisionMixin(MmprojModel):
-    """LFM3-VL vision tower and projector, shared with LFM3-Omni."""
-
-    def _set_lfm3vl_gguf_parameters(self):
-        assert self.hparams_vision is not None
-        # clip.cpp hardcodes the GELU projector and its input norm eps
-        if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
-                or self.global_config["projector_layer_norm_eps"] != 1e-6):
-            raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
-        # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
-        if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
-                or self.hparams_vision["conv_kernel_size"] != 3):
-            raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
-        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.LFM3VL)
-        self.gguf_writer.add_vision_head_count_kv(self.hparams_vision["num_key_value_heads"])
-        self.gguf_writer.add_vision_spatial_merge_size(self.global_config["downsample_factor"])
-        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["norm_eps"])
-        self.gguf_writer.add_vision_use_silu(True)
-        size = self.preprocessor_config["size"]
-        self.gguf_writer.add_vision_min_pixels(size["shortest_edge"])
-        self.gguf_writer.add_vision_max_pixels(size["longest_edge"])
-        # PIL resample enum, the torchvision backend has no lanczos
-        resample = self.preprocessor_config["resample"]
-        resize_algo = {2: "bilinear", 3: "bicubic"}.get(resample)
-        if resize_algo is None:
-            raise ValueError(f"unsupported resample: {resample}")
-        self.gguf_writer.add_vision_image_resize_algo(resize_algo)
-        self._check_video_config()
-
-    # mtmd_get_video_ts_style and the lfm3vl video frame sizing hardcode these, conversion fails on a mismatch
-    _video = {"fps": 2.0, "min_frames": 4, "temporal_patch_size": 2, "size": {"shortest_edge": 4096, "longest_edge": 25165824}}
-    _video_max_frames = 768
-
-    def _check_video_config(self):
-        import json
-        video = self.preprocessor_config.get("video_processor")
-        path = self.dir_model / "video_preprocessor_config.json"
-        if video is None and path.is_file():
-            with open(path, encoding="utf-8") as f:
-                video = json.load(f)
-        if video is None:
-            return
-        for k, v in {**self._video, "max_frames": self._video_max_frames}.items():
-            if video.get(k) != v:
-                raise ValueError(f"unsupported video {k}={video.get(k)!r}, expected {v!r}")
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if "patch_embd" in new_name:
-            return gguf.GGMLQuantizationType.F16
-        if "conv2d_spatial" in new_name:
-            # CUDA CONV_2D_DW requires an f32 kernel; the tensor is tiny
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def _lfm3vl_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "model.vision_tower.patchifier.weight":
-            # qwen2vl-style layout [channel, temporal, ph, pw], one conv kernel per temporal slice
-            assert self.hparams_vision is not None
-            t = self.hparams_vision["temporal_patch_size"]
-            p = self.hparams_vision["patch_size"]
-            n_embd = data_torch.shape[0]
-            w = data_torch.view(n_embd, 3, t, p, p)
-            yield ("v.patch_embd.weight", w[:, :, 0].contiguous())
-            if t == 2:
-                yield ("v.patch_embd.weight.1", w[:, :, 1].contiguous())
-            else:
-                assert t == 1, f"unsupported vision_temporal_patch_size {t}"
-            return
-        if name == "model.vision_tower.embedding_norm.weight":
-            yield ("v.embedding_norm.weight", data_torch)
-            return
-        if "conv.in_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_in_proj.weight", data_torch)
-            return
-        if "conv.spatial_conv.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_spatial.weight", data_torch)
-            return
-        if "conv.out_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_out_proj.weight", data_torch)
-            return
-
-        # projector: input LayerNorm -> linear_1 -> GELU -> linear_2
-        if name.startswith("model.multi_modal_projector."):
-            layer, suffix = name.removeprefix("model.multi_modal_projector.").split(".")
-            yield ({"layer_norm": "mm.input_norm", "linear_1": "mm.0", "linear_2": "mm.2"}[layer] + "." + suffix, data_torch)
-            return
-
-        yield (self.map_tensor_name(name), data_torch)
-
-
-@ModelBase.register("Lfm3VlForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-VL-2B")
-class LFM3VLMmprojModel(LFM3VLVisionMixin):
-    has_vision_encoder = True
-    has_audio_encoder = False
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-
-
-def _is_lfm3_audio_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("model_type") == "lfm3_audio" and "lfm" in config
-
-
-# read the release config.json as is: a transformers that knows lfm3_audio rewrites it into another layout
-@ModelBase.register_hparams_loader(_is_lfm3_audio_checkpoint)
-def _load_lfm3_audio_hparams(dir_model: Path) -> dict[str, Any]:
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    config["text_config"] = config["lfm"]
-    return config
-
-
-@ModelBase.register("Lfm3AudioForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Audio-2B")
-class LFM3AudioMmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    # the downsampler has the LFM2-Audio pre-encoder layout: conv, dw conv, pw conv, dw conv, pw conv
-    _downsample_idx = {"conv_in": 0, "conv_mid": 2, "lin_mid": 3, "conv_end": 5, "lin_end": 6}
-
-    # audio output: MaskGIT depthformer (a.gen.code.*) and the codec decoder (a.gen.wav.*)
-    _depth_layer_map = {
-        "operator_norm":      gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
-        "self_attn.q_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q,
-        "self_attn.q_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,
-        "self_attn.k_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K,
-        "self_attn.k_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,
-        "self_attn.v_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V,
-        "self_attn.o_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
-        "self_attn.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
-        "ffn_norm":           gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
-        "feed_forward.w1":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
-        "feed_forward.w3":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
-        "feed_forward.w2":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
-    }
-    _codec_layer_map = {
-        "operator_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM,
-        "post_operator_norm":    gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM,
-        "self_attn.q_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q,
-        "self_attn.q_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM,
-        "self_attn.k_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K,
-        "self_attn.k_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM,
-        "self_attn.v_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V,
-        "self_attn.o_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT,
-        "self_attn.gate_proj":   gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE,
-        "conv.in_proj":          gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1,
-        "conv.conv":             gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW,
-        "conv.out_proj":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2,
-        "ffn_norm":              gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM,
-        "post_ffn_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM,
-        "feed_forward.w1":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE,
-        "feed_forward.w3":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
-        "feed_forward.w2":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
-    }
-    # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
-    _codec = {
-        "model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256,
-        "encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6,
-        "decoder_window": 30, "sampling_rate": 24000,
-    }
-    # the lfm3a pipeline in mtmd-helper-gen.cpp hardcodes these
-    _generation = {"codebooks": 12, "maskgit_schedule": [4, 4, 4], "interleaved_n_text": 6, "interleaved_n_audio": 12}
-
-    # speech-code sampling of the model card, generation_config.json overrides it
-    _audio_sampling = {
-        "audio_temperature": 1.0, "audio_top_k": 128, "audio_repetition_penalty": 1.1, "audio_repetition_window": 4096,
-    }
-
-    # mtmd_audio_preprocessor_lfm3a hardcodes this front end, conversion fails on a mismatch
-    _feature_extractor = {
-        "sampling_rate": 16000, "feature_size": 128, "n_fft": 400, "win_length": 400, "hop_length": 160,
-        "log_floor": -12.0, "normalization_offset": 4.0, "normalization_divisor": 4.0, "encoder_stride": 8,
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # audio output is exported when the checkpoint ships the codec
-        self._audio_output = (self.dir_model / "audio_tokenizer").is_dir()
-        gen = self.get_generation_config()
-        self._n_codebooks: int = gen["codebooks"]
-        self._audio_vocab: int = gen["audio_vocab_size"]
-        self._depth_embd: dict[int, Tensor] = {}
-        self._depth_norm: dict[int, Tensor] = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["encoder"]
-        cfg = dict(enc["lfm"])
-        cfg["num_mel_bins"] = enc["dim_in"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        steps = c["maskgit_steps"]
-        return {
-            "codebooks": c["codebooks"], "audio_vocab_size": c["audio_vocab_size"],
-            "maskgit_schedule": [c["codebooks"] // steps] * steps,
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depthformer"]
-
-    def set_gguf_parameters(self):
-        assert self.hparams_audio is not None
-        fe = self.preprocessor_config["feature_extractor"]
-        if fe["feature_extractor_type"] not in ("Lfm3AudioFeatureExtractor", "Lfm3OmniFeatureExtractor"):
-            raise ValueError(f"unsupported feature_extractor_type {fe['feature_extractor_type']!r}")
-        for k, v in self._feature_extractor.items():
-            if fe[k] != v:
-                raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}")
-
-        # clip.cpp hardcodes the encoder rope base
-        if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported audio encoder rope_theta")
-
-        self.hparams_audio["intermediate_size"] = lfm2_feed_forward_length(self.hparams_audio)
-        super().set_gguf_parameters()
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.LFM3A)
-        self.gguf_writer.add_audio_head_count_kv(self.hparams_audio["num_key_value_heads"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio["norm_eps"])
-        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])
-        self.gguf_writer.add_audio_window_size(self.hparams_audio["sliding_window"])
-
-        if not self._audio_output:
-            return
-
-        # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp
-        gen = self.get_generation_config()
-        for k, v in self._generation.items():
-            if gen[k] != v:
-                raise ValueError(f"unsupported {k}={gen[k]!r}, expected {v!r}")
-        depth = self.get_depth_config()
-        if depth.get("is_causal", False) or depth["use_sandwich_norm"] or set(depth["layer_types"]) != {"full_attention"}:
-            raise ValueError("unsupported depthformer, expected a bidirectional all-attention stack")
-        if depth["rope_parameters"]["full_attention"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported depthformer rope_theta")
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.LFM3A_GEN)
-        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_gen_audio_embedding_length(depth["hidden_size"])
-        self.gguf_writer.add_gen_audio_feed_forward_length(depth["intermediate_size"])
-        self.gguf_writer.add_gen_audio_block_count(depth["num_hidden_layers"])
-        self.gguf_writer.add_gen_audio_head_count(depth["num_attention_heads"])
-        self.gguf_writer.add_gen_audio_head_count_kv(depth["num_key_value_heads"])
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(depth["norm_eps"])
-
-        # sampling defaults of the speech codes, next to the text ones that become general.sampling.*
-        gen = gguf.Metadata.load_generation_config(self.dir_model)
-        for key, add in (("audio_temperature",        self.gguf_writer.add_gen_audio_sampling_temp),
-                         ("audio_top_k",              self.gguf_writer.add_gen_audio_sampling_top_k),
-                         ("audio_repetition_penalty", self.gguf_writer.add_gen_audio_sampling_penalty_repeat),
-                         ("audio_repetition_window",  self.gguf_writer.add_gen_audio_sampling_penalty_last_n)):
-            value = gen.get(key, self._audio_sampling.get(key))
-            if value is not None:
-                add(value)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if new_name.startswith("a.conv1d.") or "conv_dw" in new_name or new_name.startswith("a.gen.wav.quant."):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if self._audio_output:
-            yield from self._codec_tensors()
-
-    def _codec_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        import json
-        from safetensors.torch import load_file
-
-        T = gguf.MODEL_TENSOR
-        codec_dir = self.dir_model / "audio_tokenizer"
-        with open(codec_dir / "config.json", encoding="utf-8") as f:
-            cfg = json.load(f)
-        for k, v in self._codec.items():
-            if cfg.get(k) != v:
-                raise ValueError(f"unsupported audio_tokenizer {k}={cfg.get(k)!r}, expected {v!r}")
-        if math.prod(cfg["fsq_levels"]) != self._audio_vocab - 1:
-            raise ValueError(f"audio_vocab_size {self._audio_vocab} does not match the codec, expected codebook size + 1")
-
-        state_dict = load_file(codec_dir / "model.safetensors")
-
-        # FSQ code table: index -> the len(levels) quantized values, see FiniteScalarQuantization._indices_to_codes
-        levels = torch.tensor(cfg["fsq_levels"], dtype=torch.int64)
-        basis = torch.cumprod(torch.cat([torch.ones(1, dtype=torch.int64), levels[:-1]]), dim=0)
-        half = levels // 2
-        idx = torch.arange(int(levels.prod()), dtype=torch.int64)[:, None]
-        codes = ((idx // basis) % levels - half) / half
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), codes.to(torch.float32))
-
-        # one projection over the concatenated code vectors of all codebooks, under quantizer.fsq. in newer exports
-        quantizer = "quantizer.fsq." if "quantizer.fsq.project_out.weight" in state_dict else "quantizer."
-        for suffix in ("weight", "bias"):
-            yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT, suffix=f".{suffix}"), state_dict[f"{quantizer}project_out.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=f".{suffix}"), state_dict[f"from_quantizer.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=f".{suffix}"), state_dict[f"decoder.lin.{suffix}"])
-        yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), state_dict["decoder.lfm.embedding_norm.weight"])
-
-        for name, data_torch in state_dict.items():
-            if not name.startswith("decoder.lfm.layers."):
-                continue
-            bid_str, key = name.removeprefix("decoder.lfm.layers.").rsplit(".weight", 1)[0].split(".", 1)
-            if key == "conv.conv":
-                data_torch = data_torch.squeeze(1)
-            yield (self.format_tensor_name(self._codec_layer_map[key], int(bid_str)), data_torch)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith("a.gen.wav."):
-            yield (name, data_torch)
-            return
-
-        if name.startswith(("depthformer.", "depth_", "codebook_embedding.", "mask_embedding")):
-            if self._audio_output:
-                yield from self._depthformer_tensor(data_torch, name, bid)
-            return
-
-        if not name.startswith(("encoder.", "mlp.")):
-            return
-
-        if name.startswith("encoder.downsample."):
-            layer, suffix = name.removeprefix("encoder.downsample.").split(".")
-            if layer == "lin_out":
-                if suffix == "weight":
-                    data_torch = data_torch.squeeze(-1)
-                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_OUT, suffix=f".{suffix}"), data_torch)
-                return
-            if suffix == "bias":
-                data_torch = data_torch[:, None, None]
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_CONV1D, self._downsample_idx[layer], f".{suffix}"), data_torch)
-            return
-
-        # conv op requires 2d tensor
-        if name.endswith("conv.conv.weight"):
-            data_torch = data_torch.squeeze(1)
-
-        yield (self.map_tensor_name(name), data_torch)
-
-    def _depthformer_tensor(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        T = gguf.MODEL_TENSOR
-
-        if name.startswith("depthformer.model.layers."):
-            key = name.removeprefix("depthformer.model.layers.").split(".", 1)[1].rsplit(".weight", 1)[0]
-            yield (self.format_tensor_name(self._depth_layer_map[key], bid), data_torch)
-            return
-        if name == "depthformer.model.embedding_norm.weight":
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
-            return
-        if name.startswith("depth_linear."):
-            yield (self.format_tensor_name(T.A_GEN_CODE_PROJ_IN, suffix="." + name.rsplit(".", 1)[1]), data_torch)
-            return
-        if name == "mask_embedding":
-            yield (self.format_tensor_name(T.A_GEN_CODE_MASK_EMBD, suffix=""), data_torch)
-            return
-        if name == "codebook_embedding.weight":
-            assert data_torch.shape[0] == self._n_codebooks * self._audio_vocab
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch)
-            return
-
-        # per-codebook heads: the embedding table is tied to to_logits, merge the 12 tables into one tensor
-        assert name.startswith("depth_embeddings.")
-        idx = int(name.split(".")[1])
-        key = name.split(".", 2)[2]
-        if key == "embedding.weight":
-            self._depth_embd[idx] = data_torch
-            if len(self._depth_embd) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), torch.stack([self._depth_embd[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "embedding_norm.weight":
-            self._depth_norm[idx] = data_torch
-            if len(self._depth_norm) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_HEAD_NORM), torch.stack([self._depth_norm[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "to_logits.weight":
-            if not torch.equal(data_torch, self._depth_embd[idx]):
-                raise ValueError(f"depth_embeddings.{idx} is not tied")
-        else:
-            raise ValueError(f"unexpected tensor {name}")
-
-
-@ModelBase.register("Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Omni-2B")
-class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel):
-    has_vision_encoder = True
-    has_audio_encoder = True
-
-    _audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64}
-    _video_max_frames = 384
-
-    # the audio side is the LFM3-Audio model under the HF-native names
-    _audio_renames = (
-        ("model.audio_encoder.lfm.", "encoder.lfm.lfm."),
-        ("model.audio_encoder.", "encoder."),
-        ("model.audio_projector.layer_norm.", "mlp.model.0."),
-        ("model.audio_projector.linear_1.", "mlp.model.1."),
-        ("model.audio_projector.linear_2.", "mlp.model.3."),
-        ("model.depth_decoder.model.", "depthformer.model."),
-        ("model.depth_decoder.", ""),
-        ("model.codebook_embedding.", "codebook_embedding."),
-    )
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["audio_encoder_config"]
-        cfg = dict(enc["text_config"])
-        cfg["num_mel_bins"] = enc["num_mel_bins"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        return {
-            "codebooks": c["num_codebooks"], "audio_vocab_size": c["codebook_size"], "maskgit_schedule": c["maskgit_schedule"],
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depth_config"]
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-            return
-        for old, new in self._audio_renames:
-            if name.startswith(old):
-                yield from super().modify_tensors(data_torch, new + name.removeprefix(old), bid)
-                return
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/limite.py

@@ -1,184 +0,0 @@
-from __future__ import annotations
-
-from typing import Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, TextModel, gguf
-
-
-@ModelBase.register("LimiteForCausalLM")
-@ModelBase.example("paradigma-inc/limite-1b-violetto")
-class LimiteModel(TextModel):
-    model_arch = gguf.MODEL_ARCH.LIMITE
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hp = self.hparams
-
-        expected = {
-            "pos_mode": "rope",
-            "qk_norm": "rms_pre_rope",
-            "rope_style": "interleaved_pairs_odd_lane_sign_flip",
-            "rope_per_layer": False,
-            "rms_norm_has_weight": False,
-            "rms_norm_eps_mode": "torch_finfo_default",
-            "mlp_type": "swiglu",
-            "attn_gate_applied": "per_head_before_o_proj",
-            "ve_head_slice": "first_num_key_value_heads",
-            "ve_applied_before_qk_norm": True,
-            "global_nope": True,
-            "global_window": -1,
-            "mudd_r_site": "resid",
-            "tie_word_embeddings": True,
-            "final_softcap": 0.0,
-        }
-        for key, value in expected.items():
-            if hp.get(key) != value:
-                raise NotImplementedError(f"Limite: unsupported {key}={hp.get(key)!r}, expected {value!r}")
-        if hp["softcap_logits"]["kind"] != "sigmoid":
-            raise NotImplementedError(f"Limite: unsupported softcap_logits kind {hp['softcap_logits']['kind']!r}")
-
-        self.n_head    = hp["num_attention_heads"]
-        self.n_head_kv = hp["num_key_value_heads"]
-        self.head_dim  = hp["head_dim"]
-        self.n_pairs   = hp["rope_n_pairs"]
-
-        self.mudd_taps: dict[int, list[int]] = {}
-        if hp.get("mudd"):
-            if not hp.get("mudd_mlp"):
-                raise NotImplementedError("Limite: mudd without mudd_mlp is not supported")
-            self.mudd_taps = {int(il): [int(t) for t in taps] for il, taps in hp["mudd_tap_idx"].items()}
-            assert all(len(taps) == hp["mudd_taps"] for taps in self.mudd_taps.values())
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        hp = self.hparams
-
-        self.gguf_writer.add_key_length(self.head_dim)
-        self.gguf_writer.add_value_length(self.head_dim)
-
-        # torch rms_norm with eps=None uses the fp32 epsilon, even for bf16 inputs
-        self.gguf_writer.add_layer_norm_rms_eps(torch.finfo(torch.float32).eps)
-        self.gguf_writer.add_attention_scale(hp["attention_softmax_scale"])
-        self.gguf_writer.add_xsa_eps(hp["xsa_normalize_eps"] if hp.get("xsa") else 0.0)
-
-        # HF freq_i = base^(-i/(n_pairs-1)), ggml uses freq_base^(-i/n_pairs)
-        self.gguf_writer.add_rope_dimension_count(2 * self.n_pairs)
-        self.gguf_writer.add_rope_freq_base(hp["rope_base_local"] ** (self.n_pairs / (self.n_pairs - 1)))
-
-        # HF keeps keys k >= q - window, so a window of W spans W + 1 tokens
-        global_layers = set(hp["global_layers"])
-        self.gguf_writer.add_sliding_window(hp["sliding_window"] + 1)
-        self.gguf_writer.add_sliding_window_pattern([il not in global_layers for il in range(self.block_count)])
-
-        cap = hp["softcap_logits"]
-        self.gguf_writer.add_final_logit_sigmoid_capping(cap["a"], cap["b"], cap["c"])
-
-        if self.mudd_taps:
-            n_taps = hp["mudd_taps"]
-            tap_indices = [-1] * (self.block_count * n_taps)
-            for il, taps in self.mudd_taps.items():
-                tap_indices[il * n_taps:(il + 1) * n_taps] = taps
-            self.gguf_writer.add_mudd_feed_forward_length(hp["mudd_inter"])
-            self.gguf_writer.add_mudd_tap_count(n_taps)
-            self.gguf_writer.add_mudd_tap_indices(tap_indices)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # tiny mixing weights, kept at full precision as in the reference
-        # the mudd tensors come from a global source tensor, so take the block id from the new name
-        if new_name.startswith("blk."):
-            bid = int(new_name.split(".")[1])
-        for key in (gguf.MODEL_TENSOR.ATTN_GATE, gguf.MODEL_TENSOR.ATTN_VE_GATE, gguf.MODEL_TENSOR.MUDD_DOWN, gguf.MODEL_TENSOR.MUDD_UP):
-            if self.match_model_tensor_name(new_name, key, bid):
-                return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def _swap_rope_pairs(self, w: Tensor, n_head: int) -> Tensor:
-        # HF rotates each pair by -theta; swapping the pair lanes gives the ggml +theta rotation.
-        # q and k are both swapped, so the attention scores do not change.
-        idx = torch.arange(self.head_dim)
-        idx[:2 * self.n_pairs] = idx[:2 * self.n_pairs].view(-1, 2).flip(-1).reshape(-1)
-        w = w.reshape(n_head, self.head_dim, -1)
-        return w[:, idx, :].reshape(n_head * self.head_dim, -1)
-
-    def _pad_cols(self, w: Tensor) -> Tensor:
-        # gates read only the first channels of the input, pad them to n_embd
-        return torch.nn.functional.pad(w, (0, self.hparams["hidden_size"] - w.shape[-1]))
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        hp = self.hparams
-        T = gguf.MODEL_TENSOR
-
-        def load(n: str) -> Tensor:
-            return self.model_tensors[n]().float()
-
-        if name == "model.value_embeds.weight":
-            ve = data_torch.reshape(data_torch.shape[0], -1, hp["ve_dim"])[:, :self.n_head_kv]
-            ve = torch.nn.functional.pad(ve, (0, self.head_dim - hp["ve_dim"]))
-            yield self.format_tensor_name(T.VALUE_EMBD), ve.reshape(ve.shape[0], -1) * hp["ve_gate_scale"]
-            return
-
-        if name == "model.mudd.dense1":
-            for il in self.mudd_taps:
-                yield self.format_tensor_name(T.MUDD_DOWN, il), data_torch
-            return
-        if name == "model.mudd.dense2":
-            # rows [0, n) mix the attention input, rows [n, 2n) mix the residual
-            dense2_mlp = load("model.mudd.dense2_mlp")
-            bias       = load("model.mudd.bias")
-            bias_mlp   = load("model.mudd.bias_mlp")
-            for il, taps in self.mudd_taps.items():
-                n = len(taps)
-                w = torch.cat([data_torch[il, :n], dense2_mlp[il, :n]], 0)
-                b = torch.cat([bias[il, :n], bias_mlp[il, :n]], 0)
-                yield self.format_tensor_name(T.MUDD_UP, il), w
-                yield self.format_tensor_name(T.MUDD_UP, il, ".bias"), b
-            return
-        if name.startswith("model.mudd."):
-            return
-
-        if bid is None:
-            yield from super().modify_tensors(data_torch, name, bid)
-            return
-
-        prefix = f"model.layers.{bid}."
-        name = name.removeprefix(prefix)
-
-        if name in ("self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"):
-            data_torch = data_torch * load(prefix + "self_attn.qkv_scale")
-            if name == "self_attn.q_proj.weight":
-                data_torch = self._swap_rope_pairs(data_torch, self.n_head)
-            elif name == "self_attn.k_proj.weight":
-                data_torch = self._swap_rope_pairs(data_torch, self.n_head_kv)
-        elif name == "self_attn.o_proj.weight":
-            data_torch = data_torch * load(prefix + "self_attn.o_scale") * load(prefix + "post_lambda_attn") * hp["attn_gate_scale"]
-        elif name == "mlp.down_proj.weight":
-            data_torch = data_torch * load(prefix + "post_lambda_mlp")
-        elif name == "self_attn.attn_gate":
-            yield self.format_tensor_name(T.ATTN_GATE, bid), self._pad_cols(data_torch)
-            return
-        elif name == "self_attn.ve_gate":
-            yield self.format_tensor_name(T.ATTN_VE_GATE, bid), self._pad_cols(data_torch[:self.n_head_kv])
-            return
-        elif name == "self_attn.xsa_alpha":
-            if hp.get("xsa") and bid in hp["xsa_layers"]:
-                yield self.format_tensor_name(T.ATTN_XSA_ALPHA, bid), torch.tanh(data_torch)
-            return
-        elif name == "resid_lambda_attn":
-            yield self.format_tensor_name(T.ATTN_RESID_SCALE, bid), data_torch.reshape(1)
-            return
-        elif name == "resid_lambda_mlp":
-            yield self.format_tensor_name(T.FFN_RESID_SCALE, bid), data_torch.reshape(1)
-            return
-        elif name in ("self_attn.qkv_scale", "self_attn.o_scale", "post_lambda_attn", "post_lambda_mlp"):
-            # folded into the projections above
-            return
-
-        yield from super().modify_tensors(data_torch, prefix + name, bid)

conversion/metis.py

@@ -1,204 +0,0 @@
-from __future__ import annotations
-
-import math
-from typing import Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import LazyTorchTensor, ModelBase, TextModel, gguf
-
-
-@ModelBase.register("Metis16ForCausalLM")
-class Metis16Model(TextModel):
-    model_arch = gguf.MODEL_ARCH.METIS16
-
-    # read by generate_extra_tensors() or not used at inference: dropped in modify_tensors()
-    _folded = {
-        "pass_embeddings", "lm_head.weight", "depth_budget.multiplier",
-        "depth_memory.pass_embeddings", "depth_memory.anchor_embeddings",
-        "depth_memory.state_write.bias", "depth_memory.metadata_write.bias",
-    }
-    _folded_layer = {
-        "mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits",
-        "mixer_connection.read_logits", "mixer_connection.write_logits",
-        "moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits",
-        "moe_connection.read_logits", "moe_connection.write_logits",
-        "moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier",
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hp = self.hparams
-        self.n_pass = hp["num_passes"]
-        self.attn_layers = list(hp["attention_layer_indices"])
-        self._generated: set[str] = set()
-        # one cache slot per pass and layer, while the tensors exist once per physical layer
-        self.block_count = self.n_pass * hp["num_hidden_layers"]
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-        # turns end on the first eos id, so llama.cpp must stop on it too
-        eos = self.hparams.get("eos_token_id")
-        if isinstance(eos, list) and eos:
-            self.gguf_writer.add_eot_token_id(eos[0])
-
-    def set_gguf_parameters(self):
-        hp = self.hparams
-        super().set_gguf_parameters()
-
-        n_layer = hp["num_hidden_layers"]
-        self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)])
-
-        self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"])
-        self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"])
-        self.gguf_writer.add_expert_shared_count(1)
-        self.gguf_writer.add_moe_latent_size(hp["latent_dim"])
-
-        d_inner = hp["mamba_expand"] * hp["hidden_size"]
-        self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"])
-        self.gguf_writer.add_ssm_inner_size(d_inner)
-        self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"])
-        self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"])
-        self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"])
-        self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"])
-
-        self.gguf_writer.add_hyper_connection_count(hp["n_streams"])
-        self.gguf_writer.add_metis_pass_count(self.n_pass)
-        self.gguf_writer.add_metis_memory_length(hp["memory_dim"])
-        self.gguf_writer.add_metis_route_length(hp["route_feature_dim"])
-        self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"])
-        self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"])
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        # the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it
-        if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")):
-            name += ".bias"
-        elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates"))
-              or name in ("depth_memory.stream_gate", "stream_embeddings")):
-            name += ".weight"
-        elif name == "depth_memory.stream_gate_bias":
-            name = "depth_memory.stream_gate.bias"
-        return super().filter_tensors((name, gen))
-
-    def _load(self, name: str) -> Tensor:
-        return LazyTorchTensor.to_eager(self.model_tensors[name]())
-
-    def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]:
-        self._generated.add(name)
-        return name, data
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        hp = self.hparams
-        r = hp["route_feature_dim"]
-        n_stream = hp["n_streams"]
-        T = gguf.MODEL_TENSOR
-        passes = self._load("pass_embeddings")[: self.n_pass]
-
-        # the mixes depend only on the weights and the pass, so Sinkhorn runs once here
-        # the controller runs in the checkpoint dtype, as the reference does
-        for il in range(hp["num_hidden_layers"]):
-            for src, (mix, read, write) in (
-                ("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)),
-                ("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)),
-            ):
-                prefix = f"layers.{il}.{src}."
-                ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias")
-                mixes, reads, writes = [], [], []
-                for p in range(self.n_pass):
-                    delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float()
-                    m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream])
-                    log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream)
-                    for _ in range(hp["mhc_sinkhorn_iterations"]):
-                        log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True)
-                        log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True)
-                    mixes.append(log_m.exp())
-                    reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1))
-                    writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1))
-                act = ctrl_w.dtype
-                yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float())
-                yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float())
-                yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float())
-
-        # one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width
-        meta_w = self._load("depth_memory.metadata_write.weight").float()
-        mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"]
-        base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float()
-                + mean_k * meta_w[:, r])
-        pass_e = self._load("depth_memory.pass_embeddings").float()
-        anchor_e = self._load("depth_memory.anchor_embeddings").float()
-        n_anchor = len(self.attn_layers) + 1
-        entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)])
-        yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias)
-
-    def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # the graph reads the concatenated features one block at a time; each block is its own tensor
-        for i, part in enumerate(parts):
-            piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:]
-            yield from super().modify_tensors(piece, template.format(part), bid)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name in self._generated:
-            yield name, data_torch
-            return
-        if name in self._folded:
-            return
-        hp = self.hparams
-        d = hp["hidden_size"]
-
-        if bid is None:
-            if name == "stream_embeddings.weight":
-                # the graph keeps the streams innermost: [n_stream, n_embd, n_tokens]
-                yield from super().modify_tensors(data_torch.T.contiguous(), name, bid)
-                return
-            if name == "depth_memory.metadata_write.weight":
-                # metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized
-                r = hp["route_feature_dim"]
-                meta_w = data_torch.float()
-                entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"])
-                data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1)
-            elif name == "depth_memory.stream_gate.weight":
-                data_torch = data_torch.float() / math.sqrt(d)
-            elif name == "depth_memory.route_projection.weight":
-                yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid)
-                return
-            elif name == "continuation.hidden.weight":
-                yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid)
-                return
-            yield from super().modify_tensors(data_torch, name, bid)
-            return
-
-        local = name.removeprefix(f"layers.{bid}.")
-        if local in self._folded_layer:
-            return
-
-        if local == "mixer.impl.mixer.A_log":
-            data_torch = -torch.exp(data_torch).unsqueeze(-1)
-        elif local == "mixer.impl.mixer.D":
-            data_torch = data_torch.unsqueeze(-1)
-        elif local == "mixer.impl.mixer.conv1d.weight":
-            data_torch = data_torch.squeeze(1)
-        elif local == "mixer.impl.mixer.norm.weight":
-            data_torch = data_torch.reshape(hp["mamba_n_groups"], -1)
-        elif local == "mixer.pass_gates.weight":
-            # the reference applies tanh(gate) in the checkpoint dtype
-            data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float()
-        elif local == "moe.shared_expert.gate_up.weight":
-            gate, up = data_torch.chunk(2, dim=0)
-            yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid)
-            yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        # FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32
-        if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate"))
-                or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight",
-                                      "hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)

conversion/minicpm.py

@@ -272,18 +272,3 @@ class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
             yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids]
             return
         yield from super().modify_tensors(data_torch, name, bid)
-
-
-# MiniCPM-OCR 1.0: the 4.6 stack with model_type minicpm_ocr.
-
-@ModelBase.register("MiniCPMOcrForConditionalGeneration")
-@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
-class MiniCPMOCR1_0TextModel(MiniCPMV4_6TextModel):
-    model_arch = gguf.MODEL_ARCH.QWEN35
-    no_mtp = True
-
-
-@ModelBase.register("MiniCPMOcrForConditionalGeneration")
-@ModelBase.example("openbmb/MiniCPM-OCR-1.0")
-class MiniCPMOCR1_0VisionModel(MiniCPMV4_6VisionModel):
-    projector_type = gguf.VisionProjectorType.MINICPM_OCR_1_0

conversion/mistral3.py

@@ -39,11 +39,6 @@ class Mistral3Model(TextModel):
             self.gguf_writer.add_add_bos_token(False)
             self.gguf_writer.add_add_eos_token(False)
 
-        def get_vocab_base_pre(self, tokenizer) -> str:
-            if self.origin_hf_arch == "Mistral3Model":
-                return "nemotron-3.5-embed"
-            return super().get_vocab_base_pre(tokenizer)
-
         def set_gguf_parameters(self):
             super().set_gguf_parameters()
             self._try_set_pooling_type()

conversion/mistral4.py

@@ -1,217 +0,0 @@
-from __future__ import annotations
-
-from typing import Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, gguf
-
-from .mistral import MistralModel, MistralMoeModel
-
-
-@ModelBase.register("MistralLarge4ForConditionalGeneration")
-class MistralLarge4Model(MistralMoeModel):
-    model_arch = gguf.MODEL_ARCH.MISTRAL_LARGE4
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        n_layer = self.hparams["num_hidden_layers"]
-        dsa = self.hparams["dsa"]
-
-        def expand(pattern: str) -> list[str]:
-            values = pattern.split(",")
-            return [values[i % len(values)] for i in range(n_layer)]
-
-        self.layer_has_rope = [v == "rope" for v in expand(self.hparams.get("nope") or "rope")]
-        self.layer_indexer_full = [v == "full" for v in expand(dsa.get("sharing") or "full")]
-
-        kv_sharing = self.hparams.get("kv_sharing") or {}
-        if "source_layers_str" in kv_sharing:
-            self.layer_kv_source = [int(v) for v in kv_sharing["source_layers_str"].split(",")]
-        else:
-            assert not kv_sharing, f"unsupported kv_sharing config: {kv_sharing}"
-            self.layer_kv_source = list(range(n_layer))
-        assert len(self.layer_kv_source) == n_layer
-
-        # note: the rope dim of the indexer must match the main attention
-        assert dsa.get("rope_head_dim", -1) in (-1, self.hparams["qk_rope_head_dim"])
-        assert dsa.get("match_nope", False), "indexer must follow the NoPE pattern of the main attention"
-        assert dsa.get("ssmax_scaling", False), "indexer without SSMax is not supported"
-        assert self.hparams.get("q_lora_rank") is None
-        assert not self.hparams["yarn"].get("apply_scale", True), "only YaRN without magnitude scaling is supported"
-
-    def dequant_model(self):
-        quant_config = self.hparams.get("quantization")
-        if quant_config is not None:
-            assert quant_config["qformat_weight"] == "fp8_e4m3"
-            assert quant_config.get("qscheme_weight") == "BLOCK"
-            self.hparams["quantization_config"] = {
-                "quant_method": "fp8",
-                "weight_block_size": [quant_config["block_size_out"], quant_config["block_size_in"]],
-            }
-        return super().dequant_model()
-
-    def set_gguf_parameters(self):
-        # skip the MistralMoeModel overrides, they apply the deepseek2 YaRN scaling
-        super(MistralMoeModel, self).set_gguf_parameters()
-        MistralModel.set_mistral_config(self.gguf_writer, self.hparams)
-
-        dsa = self.hparams["dsa"]
-        self.gguf_writer.add_indexer_head_count(dsa["n_heads"])
-        self.gguf_writer.add_indexer_key_length(dsa["head_dim"])
-        self.gguf_writer.add_indexer_top_k(dsa["topk"])
-        self.gguf_writer.add_indexer_types(self.layer_indexer_full)
-        self.gguf_writer.add_layer_norm_eps(dsa.get("norm_eps", 1e-6))
-
-        self.gguf_writer.add_rope_pattern(self.layer_has_rope)
-        self.gguf_writer.add_kv_source_map(self.layer_kv_source)
-        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
-        self.gguf_writer.add_final_logit_softcapping(self.hparams["softmax_tanh_gating"])
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-
-        # exported separately with --dspark
-        if name.startswith("mtp_"):
-            return None
-
-        # experts need HF names for the merging logic in DeepseekV2Model
-        # note: bare "layers.N.experts" comes from the NVFP4 quantized_layers config
-        if ".experts." in name or name.endswith(".experts"):
-            name = "model." + name.replace(".experts", ".mlp.experts")
-            name = name.replace(".w1.", ".gate_proj.").replace(".w2.", ".down_proj.").replace(".w3.", ".up_proj.")
-
-        # skip the MistralMoeModel renaming, the qscale names are needed for block FP8 dequant
-        return super(MistralMoeModel, cls).filter_tensors((name, gen))
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "lerope.alpha":
-            # LeRoPE scales inv_freq by exp(alpha), ggml divides theta by the freq factors
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.exp(-data_torch.float()))
-            return
-
-        if bid is not None and name.endswith((".ssmax_scaling.s", ".ssmax_scaling.b")):
-            tensor = gguf.MODEL_TENSOR.INDEXER_SSMAX if ".indexer." in name else gguf.MODEL_TENSOR.ATTN_SSMAX
-            suffix = ".weight" if name.endswith(".s") else ".bias"
-            yield (self.format_tensor_name(tensor, bid, suffix), data_torch.float())
-            return
-
-        if bid is not None and ".indexer.q_input_norm." in name:
-            suffix = ".weight" if name.endswith(".weight") else ".bias"
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_NORM, bid, suffix), data_torch)
-            return
-
-        if bid is not None and ".indexer." in name:
-            name = "model." + name.replace(".attention.", ".self_attn.")
-
-        # NoPE layers have a larger non-rotary head, so wkv_b is split per layer
-        if bid is not None and name.endswith(".wkv_b.weight"):
-            n_head = self.hparams["num_attention_heads"]
-            v_head_dim = self.hparams["v_head_dim"]
-            qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
-            if not self.layer_has_rope[bid]:
-                qk_nope_head_dim += self.hparams["qk_rope_head_dim"]
-
-            assert data_torch.shape[0] == n_head * (qk_nope_head_dim + v_head_dim)
-
-            kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1])
-            k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
-            k_b = k_b.transpose(1, 2)
-
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid), k_b)
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid), v_b)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-class MistralLarge4DSparkModel(MistralLarge4Model):
-    """Mistral Large 4 DSpark draft (mtp_dspark.*) as a DFLASH GGUF, export with --dspark"""
-    model_arch = gguf.MODEL_ARCH.DFLASH
-
-    _DSPARK_ROOT: dict[str, gguf.MODEL_TENSOR] = {
-        "backbone.target_projection.weight": gguf.MODEL_TENSOR.FC,
-        "backbone.target_norm.weight":       gguf.MODEL_TENSOR.ENC_OUTPUT_NORM,
-        "backbone.final_norm.weight":        gguf.MODEL_TENSOR.OUTPUT_NORM,
-        "backbone.mask_embedding":           gguf.MODEL_TENSOR.DSPARK_MASK_EMBD,
-        "autoregressive_head.embedding.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W1,
-        "autoregressive_head.output.weight":    gguf.MODEL_TENSOR.DSPARK_MARKOV_W2,
-        "autoregressive_head.joint_proj.weight": gguf.MODEL_TENSOR.DSPARK_JOINT_PROJ,
-        "autoregressive_head.joint_proj.bias":   gguf.MODEL_TENSOR.DSPARK_JOINT_PROJ,
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        mtp = self.hparams.get("mtp") or {}
-        dspark = mtp.get("dspark")
-        if dspark is None:
-            raise ValueError("Mistral Large 4 DSpark export needs the mtp.dspark config")
-        assert dspark.get("head_type") == "rnn", f"unsupported DSpark head type: {dspark.get('head_type')}"
-        assert dspark.get("draft_input") == "position_specific", f"unsupported DSpark draft input: {dspark.get('draft_input')}"
-
-        windows = {int(v) for v in (mtp.get("ragged_attention") or "").split(",") if v.strip()}
-        assert len(windows) == 1, f"DSpark draft layers must share one sliding window, got {mtp.get('ragged_attention')}"
-
-        self._n_stage = int(mtp["n_layers"])
-        self._depth = int(mtp["depth"])
-        self._window = windows.pop()
-        self._target_layers = [int(v) for v in mtp["target_activation_layers"].split(",")]
-
-        self.layer_has_rope = [True] * self._n_stage
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if not name.startswith("mtp_dspark."):
-            return None
-        name = name.removeprefix("mtp_dspark.")
-        if name.startswith("backbone.layers."):
-            return name.removeprefix("backbone."), gen
-        return f"dspark.{name}", gen
-
-    def set_vocab(self):
-        super().set_vocab()
-        # block uses mask_embd, mask token id is not used
-        self.gguf_writer.add_mask_token_id(0)
-
-    def set_gguf_parameters(self):
-        hp = self.hparams
-        w = self.gguf_writer
-        w.add_block_count(self._n_stage)
-        w.add_context_length(hp["max_position_embeddings"])
-        w.add_embedding_length(hp["hidden_size"])
-        w.add_feed_forward_length(hp["intermediate_size"])
-        w.add_head_count(hp["num_attention_heads"])
-        w.add_head_count_kv(1)
-        w.add_key_length(hp["kv_lora_rank"] + hp["qk_rope_head_dim"])
-        w.add_value_length(hp["kv_lora_rank"])
-        w.add_key_length_mla(hp["qk_nope_head_dim"] + hp["qk_rope_head_dim"])
-        w.add_value_length_mla(hp["v_head_dim"])
-        w.add_kv_lora_rank(hp["kv_lora_rank"])
-        w.add_rope_dimension_count(hp["qk_rope_head_dim"])
-        w.add_rope_freq_base(float(hp["rope_theta"]))
-        w.add_layer_norm_rms_eps(hp["rms_norm_eps"])
-        MistralModel.set_mistral_config(w, hp)
-        w.add_sliding_window(self._window)
-        w.add_final_logit_softcapping(hp["softmax_tanh_gating"])
-        w.add_block_size(self._depth)
-        # draft reads layer output = input of next layer
-        w.add_target_layers([layer + 1 for layer in self._target_layers])
-        w.add_sample_from_anchor(True)
-        w.add_has_confidence_head(False)
-        w.add_file_type(self.ftype)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith("dspark."):
-            key = name.removeprefix("dspark.")
-            if key not in self._DSPARK_ROOT:
-                raise ValueError(f"Unexpected DSpark tensor {name!r}")
-            suffix = ".bias" if key.endswith(".bias") else ".weight"
-            yield (self.format_tensor_name(self._DSPARK_ROOT[key], None, suffix), data_torch)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/nemotron.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import re
 from typing import Any, Callable, Iterable, Sequence, TYPE_CHECKING
 
 import numpy as np
@@ -368,21 +367,9 @@ class NemotronModel(TextModel):
     model_arch = gguf.MODEL_ARCH.NEMOTRON
 
     def set_vocab(self):
-        if (self.dir_model / "tokenizer.model").exists():
-            self._set_vocab_sentencepiece()
-            self.gguf_writer.add_pad_token_id(0)
-            self.gguf_writer.add_unk_token_id(1)
-        else:
-            # SPM-style BPE (Metaspace + byte fallback): mark <0xXX> tokens as BYTE so they decode to raw bytes
-            tokens, toktypes, tokpre = self.get_vocab_base()
-            toktypes = [gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-F]{2}>", tok) else typ for tok, typ in zip(tokens, toktypes)]
-            self.gguf_writer.add_tokenizer_model("gpt2")
-            self.gguf_writer.add_tokenizer_pre(tokpre)
-            self.gguf_writer.add_token_list(tokens)
-            self.gguf_writer.add_token_types(toktypes)
-
-            special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
-            special_vocab.add_to_gguf(self.gguf_writer)
+        self._set_vocab_sentencepiece()
+        self.gguf_writer.add_pad_token_id(0)
+        self.gguf_writer.add_unk_token_id(1)
 
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
@@ -396,10 +383,7 @@ class NemotronModel(TextModel):
         rot_pct = self.rope_parameters["partial_rotary_factor"]
         n_embd = self.find_hparam(["hidden_size", "n_embd"])
         n_head = self.find_hparam(["num_attention_heads", "n_head"])
-        # Use explicit head_dim when available; fallback keeps standard Nemotron working.
-        # Without this, models where n_head * head_dim != n_embd (e.g. 48 heads, head_dim=128) produce the wrong rope dimension count.
-        head_dim = self.hparams.get("head_dim") or n_embd // n_head
-        self.gguf_writer.add_rope_dimension_count(int(rot_pct * head_dim))
+        self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head)
 
         # * RopeScaling for Nemotron
         factor = self.hparams.get("factor") or self.rope_parameters.get("factor")
@@ -428,19 +412,11 @@ class NemotronHModel(GraniteHybridModel):
     is_moe: bool = False
     supports_mtp_export = True
     _experts: list[dict[str, Tensor]] | None = None
-    # tokens past the embedding rows (left out of the vocab), and special tokens that are printed as text
-    _vocab_skip: tuple[str, ...] = ()
-    _user_defined_tokens: re.Pattern[str] | None = None
 
     _SSM_LAYER_TYPES = {"mamba", "linear_attention"}
     _ATTN_LAYER_TYPES = {"attention", "full_attention"}
     _MLP_LAYER_TYPES = {"moe"}
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        if ".mixer.experts." in name:
-            name = name.replace("backbone.", "model.").replace(".mixer.experts.", ".mlp.experts.")
-        return super()._map_fp8_weight_names(name)
-
     def __init__(self, *args, **kwargs):
         # We have to determine the correct model architecture (MoE vs non-MoE) before
         # calling the parent __init__. This is because the parent constructor
@@ -619,11 +595,7 @@ class NemotronHModel(GraniteHybridModel):
         vocab_size = -(vocab_size // -pad_vocab) * pad_vocab
         self.hparams["vocab_size"] = vocab_size
 
-        # draft-model converters (DFlash, LiLiCorr) call this with their own instance, which has neither hook
-        vocab_skip = getattr(self, "_vocab_skip", ())
-        user_defined_tokens = getattr(self, "_user_defined_tokens", None)
-
-        assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size  # ty: ignore[unresolved-attribute]
+        assert max(tokenizer.vocab.values()) < vocab_size  # ty: ignore[unresolved-attribute]
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
@@ -645,8 +617,7 @@ class NemotronHModel(GraniteHybridModel):
                         if previous_token != token:
                             logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
 
-                    printed = user_defined_tokens is not None and user_defined_tokens.fullmatch(token)
-                    if (added_tokens_decoder[i].special or self.does_token_look_special(token)) and not printed:
+                    if added_tokens_decoder[i].special or self.does_token_look_special(token):
                         toktypes.append(gguf.TokenType.CONTROL)
                     else:
                         token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")  # pre-normalize user-defined spaces
@@ -842,140 +813,3 @@ class NemotronHPuzzleModel(NemotronHModel):
         if item[0].startswith("mtp."):
             return None
         return super().filter_tensors(item)
-
-
-# NeMo speechlm2 SALM checkpoint: a Nemotron-H decoder under llm.*, the speech encoder under perception.*
-@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration")
-@ModelBase.example("nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B")
-class NemotronTranscribeModel(NemotronHModel):
-    model_arch = gguf.MODEL_ARCH.NEMOTRON_H
-    # the audio placeholder (id vocab_size) has no embedding row; speaker, event and style tags print with the transcript
-    _vocab_skip = ("<|audio|>",)
-    _user_defined_tokens = re.compile(r"<spk:\d+>|<ev:[a-z_]+>|<sty:(?:stt|end):[a-z_]+>")
-    _renames = {
-        "llm.model.embed_tokens.weight": "backbone.embeddings.weight",
-        "llm.model.norm.weight":         "backbone.norm_f.weight",
-        "llm.lm_head.weight":            "lm_head.weight",
-    }
-    # grouped experts: [n_expert, n_in, n_out] -> [n_expert, n_out, n_in]
-    _grouped_experts = {"gate_and_up_projs": "up_proj", "down_projs": "down_proj"}
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if name.startswith(("perception.", "ctc_timestamp.")) or "_extra_state" in name:
-            return None
-        if not name.startswith("llm."):
-            # the FP8 and NVFP4 releases store the decoder as backbone.*
-            raise ValueError(f"unexpected tensor {name}: only the BF16 checkpoint layout (llm.*) is supported")
-        if name in cls._renames:
-            name = cls._renames[name]
-        elif name.startswith("llm.model.layers."):
-            name = "backbone.layers." + name[len("llm.model.layers."):]
-        elif name.startswith("llm.mtp.layers."):
-            name = "mtp.layers." + name[len("llm.mtp.layers."):]
-        else:
-            raise ValueError(f"unexpected decoder tensor {name}")
-        return super().filter_tensors((name.replace("._fp32_params.", "."), gen))
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        leaf = name.rsplit(".", 1)[-1]
-        if ".mixer.experts." in name and leaf in self._grouped_experts:
-            if name.startswith("mtp."):
-                assert self._mtp_bid is not None, f"{name}: MTP head is not exported"
-                bid = self._mtp_bid
-            assert bid is not None
-            data_torch = data_torch.permute(0, 2, 1).contiguous()
-            name = f"model.layers.{bid}.mlp.experts.{self._grouped_experts[leaf]}.weight"
-            yield from ModelBase.modify_tensors(self, data_torch, name, bid)
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration")
-class NemotronTranscribeAudioModel(MmprojModel):
-    # NeMo ParallelExpertEncoder: ASR and speaker transformers, speaker activity fused into the ASR states
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        asr = self.global_config["pe_encoder_config"]["asr_encoder_cfg"]
-        return {
-            "hidden_size":         asr["d_model"],
-            "intermediate_size":   int(float(asr.get("ff_expansion", 4)) * asr["d_model"]),
-            "num_hidden_layers":   asr["n_layers"],
-            "num_attention_heads": asr["n_heads"],
-        }
-
-    @staticmethod
-    def _rope_dims(cfg: dict[str, Any]) -> int:
-        assert cfg.get("self_attention_model") == "rope" and cfg.get("subsampling", "feature_stacking") == "feature_stacking"
-        assert cfg.get("attn_mode", "full") == "full" and not cfg.get("xscaling", False) and cfg.get("pre_block_norm", True)
-        assert float(cfg.get("rope_base", 10000.0)) == 10000.0
-        n_rot = int(cfg["d_model"] // cfg["n_heads"] * float(cfg.get("rotary_fraction", 1.0)))
-        assert n_rot >= 2 and n_rot % 2 == 0
-        return n_rot
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        pe = self.global_config["pe_encoder_config"]
-        asr = pe["asr_encoder_cfg"]
-        diar = pe["diarization_model_cfg"]
-        spk = diar["encoder"]
-        assert pe.get("asr_encoder_type") == "transformer" and pe.get("speaker_feature_mode", "thresholded") == "thresholded"
-        assert pe.get("asr_normalize_type") == "per_feature" and pe.get("diar_normalize_type") in (None, "NA")
-        assert asr["subsampling_factor"] == spk["subsampling_factor"] == 8 and asr["feat_in"] == spk["feat_in"]
-        assert self._rope_dims(spk) == spk["d_model"] // spk["n_heads"] and not spk.get("qkv_bias", False)
-        assert diar.get("high_resolution", False)
-        # constants in clip: the ParallelExpertEncoder online settings and the Sortformer streaming config
-        assert (pe.get("online_inference_length", 500), pe.get("chunk_left_context", 50), pe.get("chunk_right_context", 50),
-                pe.get("diar_spkcache_len", 188), pe.get("diar_fifo_len", 40), pe.get("diar_spkcache_update_period", 300),
-                pe.get("speaker_activity_threshold", 0.5)) == (500, 50, 50, 188, 40, 300, 0.5), \
-            "streaming settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp"
-        sm = diar["sortformer_modules"]
-        assert (sm["spkcache_sil_frames_per_spk"], sm["pred_score_threshold"], sm["scores_boost_latest"], sm["strong_boost_rate"],
-                sm["weak_boost_rate"], sm["min_pos_scores_rate"], sm["use_learnable_sil_emb"]) == (1, 0.25, 0.05, 0.75, 1.5, 0.5, True), \
-            "speaker cache settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp"
-
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_TRANSCRIBE)
-        self.gguf_writer.add_audio_num_mel_bins(asr["feat_in"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-        self.gguf_writer.add_audio_subsampling_factor(asr["subsampling_factor"])
-        self.gguf_writer.add_audio_rope_dims(self._rope_dims(asr))
-        self.gguf_writer.add_audio_spk_block_count(spk["n_layers"])
-        self.gguf_writer.add_audio_spk_head_count(spk["n_heads"])
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # ParallelExpertEncoder._build_sinusoid_position_encoding
-        pe = self.global_config["pe_encoder_config"]
-        n_spk = int(pe["diarization_model_cfg"]["sortformer_modules"]["num_spks"])
-        d = int(pe["asr_encoder_cfg"]["d_model"])
-        scale = float(pe.get("spk_kernel_scale", 1.0))
-        pos = torch.arange(n_spk, dtype=torch.float32)[:, None]
-        div = torch.exp(torch.arange(0, d, 2, dtype=torch.float32) * -(np.log(10000.0) / d))
-        kernel = torch.zeros(n_spk, d, dtype=torch.float32)
-        kernel[:, 0::2] = torch.sin(pos * div)
-        kernel[:, 1::2] = torch.cos(pos * div)
-        # stored as [d, n_spk]: the graph multiplies it with the speaker activity directly
-        yield self.format_tensor_name(gguf.MODEL_TENSOR.A_FUSION_SPK_KERNEL, suffix=""), (kernel * scale).T.contiguous()
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE, None):
-            return gguf.GGMLQuantizationType.F16  # ggml_conv_1d kernel
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if not name.startswith("perception.") or "_extra_state" in name:
-            return None
-        # training-only head; the speaker model's unused front end
-        if name.startswith(("perception.encoder.diarization_model.sortformer_modules.hidden_to_spks.",
-                            "perception.encoder.diarization_model.preprocessor.")):
-            return None
-        return name, gen
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "perception.preprocessor.featurizer.fb":
-            data_torch = data_torch.squeeze(0)
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/pockettts.py

@@ -5,7 +5,6 @@ from pathlib import Path
 from typing import Any, Iterable, TYPE_CHECKING
 
 import torch
-import torch.nn.functional as F
 
 if TYPE_CHECKING:
     from torch import Tensor
@@ -46,102 +45,6 @@ def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]:
         return {name: tuple(part[name].shape) for name in part.keys()}
 
 
-def _has_voice_encoder(dir_model: Path) -> bool:
-    # without voice cloning, the mimi encoder weights are all zero
-    part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors")
-    with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part:
-        nonzero = {bool(part[name].mmap_bytes().any()) for name in part.keys() if name.startswith("mimi.encoder")}
-    if len(nonzero) != 1:
-        raise ValueError(f"{dir_model}: the mimi encoder is partly zeroed out")
-    return nonzero.pop()
-
-
-def _load_safetensors(path: Path, prefixes: tuple[str, ...]) -> dict[str, Tensor]:
-    dtypes = {"F32": torch.float32, "BF16": torch.bfloat16, "I64": torch.int64}
-    with gguf.utility.SafetensorsLocal(path) as part:
-        return {name: torch.from_numpy(part[name].mmap_bytes()).view(dtypes[part[name].dtype]).reshape(part[name].shape)
-                for name in part.keys() if name.startswith(prefixes)}
-
-
-def _rope_pairs(x: Tensor, n_head: int, rope_theta: float) -> Tensor:
-    # ggml NORM mode rope, x: [n_pos, n_embd]
-    n_pos, n_embd = x.shape
-    head_dim = n_embd // n_head
-    angle = torch.arange(n_pos, dtype=x.dtype)[:, None, None] * \
-        torch.pow(rope_theta, -torch.arange(0, head_dim, 2, dtype=x.dtype) / head_dim)
-    x = x.view(n_pos, n_head, head_dim // 2, 2)
-    return torch.stack([x[..., 0] * angle.cos() - x[..., 1] * angle.sin(),
-                        x[..., 0] * angle.sin() + x[..., 1] * angle.cos()], -1).reshape(n_pos, n_embd)
-
-
-def _voice_prompt_embd(weights: dict[str, Tensor], kv: list[Tensor], n_head: int, rope_theta: float) -> tuple[Tensor, float]:
-    # the voice state only has K/V of each layer (kv[il] is [k/v, pos, n_embd]): layer 0 V gives each frame up to its scale, layer 1 V gives the scale
-    # returns the frames without bos_before_voice, and the max relative K/V error when the frames are prompted again
-    w = {name: t.double() for name, t in weights.items()}
-    kv = [t.double() for t in kv]
-    k0, v0, v1 = kv[0][0], kv[0][1], kv[1][1]
-    n_pos, n_embd = v0.shape
-    head_dim = n_embd // n_head
-
-    def p(il: int, name: str) -> Tensor:
-        return w[f"flow_lm.transformer.layers.{il}.{name}"]
-
-    def norm(x: Tensor, il: int, name: str) -> Tensor:
-        return F.layer_norm(x, (n_embd,), p(il, f"{name}.weight"), p(il, f"{name}.bias"), 1e-5)
-
-    def heads(t: Tensor) -> Tensor:
-        return t.reshape(n_pos, n_head, head_dim).transpose(0, 1)
-
-    def rows(t: Tensor) -> Tensor:
-        return t.transpose(0, 1).reshape(n_pos, n_embd)
-
-    def attn(il: int, h: Tensor, k: Tensor, v: Tensor) -> Tensor:
-        q = _rope_pairs(h @ p(il, "self_attn.in_proj.weight").chunk(3)[0].T, n_head, rope_theta)
-        return rows(F.scaled_dot_product_attention(heads(q), heads(k), heads(v), is_causal=True)) @ p(il, "self_attn.out_proj.weight").T
-
-    def ffn(il: int, x: Tensor) -> Tensor:
-        return F.gelu(norm(x, il, "norm2") @ p(il, "linear1.weight").T) @ p(il, "linear2.weight").T
-
-    wv = p(0, "self_attn.in_proj.weight").chunk(3)[2]
-    h = torch.linalg.solve(wv, v0.T).T                   # norm1(x)
-    z = (h - p(0, "norm1.bias")) / p(0, "norm1.weight")  # (x - mean(x)) / std(x)
-
-    # fit z on [speaker_proj, 1] to get the latent direction back
-    proj = w["flow_lm.speaker_proj_weight"]
-    fit = torch.linalg.lstsq(torch.cat([proj, torch.ones(n_embd, 1, dtype=proj.dtype)], 1), z[1:].T).solution
-    frames = (proj @ fit[:-1]).T
-
-    a0 = attn(0, h, k0, v0)[1:]
-    wv1 = p(1, "self_attn.in_proj.weight").chunk(3)[2]
-
-    def err(scale: Tensor) -> Tensor:  # per frame relative error of layer 1 V
-        x = scale[:, None] * frames + a0
-        x = x + ffn(0, x)
-        return (norm(x, 1, "norm1") @ wv1.T - v1[1:]).norm(dim=1) / v1[1:].norm(dim=1)
-
-    grid = torch.logspace(-3, 3, 49, dtype=torch.float64)
-    best = torch.stack([err(s.expand(n_pos - 1)) for s in grid]).argmin(0)
-    lo, hi = grid[(best - 1).clamp(min=0)], grid[(best + 1).clamp(max=len(grid) - 1)]
-    for _ in range(40):  # golden section search, all frames at once
-        m1, m2 = hi - 0.618 * (hi - lo), lo + 0.618 * (hi - lo)
-        left = err(m1) < err(m2)
-        lo, hi = torch.where(left, lo, m1), torch.where(left, m2, hi)
-    frames = (lo + hi)[:, None] / 2 * frames
-
-    # prompt the frames again and compare K/V with the shipped state
-    x = torch.cat([w["flow_lm.bos_before_voice"].reshape(1, n_embd), frames])
-    drift = 0.0
-    for il, (k_ref, v_ref) in enumerate(kv):
-        h = norm(x, il, "norm1")
-        _, wk, wv = p(il, "self_attn.in_proj.weight").chunk(3)
-        k, v = _rope_pairs(h @ wk.T, n_head, rope_theta), h @ wv.T
-        for a, b in ((k, k_ref), (v, v_ref)):
-            drift = max(drift, ((a - b).norm(dim=1) / b.norm(dim=1)).median().item())
-        x = x + attn(il, h, k, v)
-        x = x + ffn(il, x)
-    return frames, drift
-
-
 @ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model))
 def _load_hparams(dir_model: Path) -> dict[str, Any]:
     logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes")
@@ -291,27 +194,22 @@ class PocketTTSMmprojModel(MmprojModel):
         (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V),
     )
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.has_audio_encoder = _has_voice_encoder(self.dir_model)
-
     def set_gguf_parameters(self):
         self.gguf_writer.add_file_type(self.ftype)
         assert self.hparams_audio is not None
 
         # voice-prompt encoder: mimi encoder + speaker_proj
-        if self.has_audio_encoder:
-            self.gguf_writer.add_clip_has_audio_encoder(True)
-            # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
-            self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
-            self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
-            self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
-            self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
-            self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
-            self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
-            self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-            # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
-            self.gguf_writer.add_audio_num_mel_bins(1)
+        self.gguf_writer.add_clip_has_audio_encoder(True)
+        # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
+        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
+        self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
+        self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
+        self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
+        self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
+        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+        # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
+        self.gguf_writer.add_audio_num_mel_bins(1)
 
         # generation: flow-matching decoder + mimi decoder
         # the SEANet and flow net hparams are constant across the family, clip.cpp holds them
@@ -331,57 +229,12 @@ class PocketTTSMmprojModel(MmprojModel):
         # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
         if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"):
             return gguf.GGMLQuantizationType.F16
-        # voices are read on the host
-        if new_name.startswith("voice."):
-            return gguf.GGMLQuantizationType.F32
         return False
 
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # embeddings/<name>.safetensors hold the K/V state after the voice prompt (see get_state_for_audio_prompt() in the reference), turn it back into prompt embeddings
-        paths = sorted((self.dir_model / "embeddings").glob("*.safetensors"))
-        if not paths:
-            return
-        weights = _load_safetensors(self.dir_model / "model.safetensors",
-                                    ("flow_lm.transformer.layers.", "flow_lm.speaker_proj_weight", "flow_lm.bos_before_voice"))
-        n_layer = self.global_config["num_hidden_layers"]
-        for path in paths:
-            state = _load_safetensors(path, ("transformer.layers.",))
-            kv = []
-            for il in range(n_layer):
-                prefix = f"transformer.layers.{il}.self_attn/"
-                if any(prefix + k not in state for k in ("cache", "offset")):
-                    break
-                n_pos = int(state[prefix + "offset"].item())
-                # older packs have no pad entry
-                if prefix + "pad" in state and int(state[prefix + "pad"].item()) != 0:
-                    break
-                kv.append(state[prefix + "cache"][:, 0, :n_pos].flatten(2))  # [k/v, pos, n_embd]
-            if len(kv) != n_layer or any(t.shape != kv[0].shape for t in kv) or kv[0].shape[1] < 2 or \
-                    kv[0].shape[2] != self.global_config["hidden_size"]:
-                logger.warning(f"voice {path.stem}: skipped, not an unpadded voice state of this backbone")
-                continue
-            embd, drift = _voice_prompt_embd(weights, kv, self.global_config["num_attention_heads"],
-                                             self.global_config["rope_theta"])
-            # some packs ship a state made with other weights, the reference still uses it: keep up to 50% error, warn above 5%
-            if drift > 0.5:
-                logger.warning(f"voice {path.stem}: skipped, its state does not belong to these weights (error {drift:.1e})")
-                continue
-            log = logger.warning if drift > 0.05 else logger.info
-            log(f"voice {path.stem}: {embd.shape[0]} frames, prompting them again gives the shipped state within {drift:.1e}")
-            yield (f"voice.{path.stem}", embd.float())
-
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         del bid  # the block index of the mimi transformers is parsed here, not by the base class
         T = gguf.MODEL_TENSOR
 
-        if name.startswith("voice."):
-            yield (name, data_torch)
-            return
-
-        if not self.has_audio_encoder and (name.startswith(("mimi.encoder", "mimi.downsample."))
-                                           or name == "flow_lm.speaker_proj_weight"):
-            return  # voice-prompt encoder
-
         if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"):
             return  # folded into the backbone embedding table
         if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."):

conversion/qwen.py

@@ -83,15 +83,6 @@ class Qwen2Model(TextModel):
 class Qwen2MoeModel(TextModel):
     model_arch = gguf.MODEL_ARCH.QWEN2MOE
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        if name.removesuffix(".weight").endswith(".mlp.experts.gate_up_proj"):
-            bid = next(int(part) for part in self.map_tensor_name(name).split(".") if part.isdecimal())
-            return (
-                self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid),
-                self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid),
-            )
-        return super()._map_fp8_weight_names(name)
-
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
         if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
@@ -391,16 +382,6 @@ class _QwenMtpMixin:
 class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
     model_arch = gguf.MODEL_ARCH.QWEN3NEXT
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        if name.endswith(".linear_attn.in_proj_qkvz.weight"):
-            bid = next(int(part) for part in self.map_tensor_name(name).split(".") if part.isdecimal())
-            # Both split projections keep the original per-tensor scales.
-            return (
-                self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid),
-                self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_GATE, bid),
-            )
-        return super()._map_fp8_weight_names(name)
-
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
         self.gguf_writer.add_ssm_conv_kernel(self.hparams["linear_conv_kernel_dim"])
@@ -709,52 +690,6 @@ class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self._try_set_pooling_type()
 
 
-@ModelBase.register("MiniCPMOCRForSequenceClassification")
-class MiniCPMOCRSeqClsModel(Qwen3_5TextModel):
-    # OCR consistency judge: Qwen3.5 VL trunk with a value head read at the last token
-    # the class scores are [inconsistent, consistent], the softmax temperature is 1
-    model_arch = gguf.MODEL_ARCH.QWEN35
-    no_mtp = True  # the checkpoint has no MTP head
-
-    CLASS_LABELS = ["inconsistent", "consistent"]
-
-    def set_vocab(self):
-        super().set_vocab()
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
-
-    def _systemone_template(self) -> str:
-        # the prompt of the training data, the two images are the original page and the one
-        # rendered from the OCR text; the task is fixed, there is nothing to fill in
-        # the trailing newlines are in an expression, so the template does not drop them
-        return (
-            "<|im_start|>user\n{{ images[0] }}\nOCR文本：\n{{ state }}\n{{ images[1] }}\n"
-            "第一张图是原图，第二张图是上述 OCR 文本渲染得到的图片。\n"
-            "请判断该 OCR 文本及其对应渲染图是否与原图内容一致。\n"
-            "这是一个二分类任务：一致标为 1，不一致标为 0。"
-            "{{ '<|im_end|>\\n<|im_start|>assistant\\n<think>\\n\\n</think>\\n\\n' }}"
-        )
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        # the config.json of the checkpoint keeps the one of the base model, it has no num_labels / id2label
-        if "score.weight" not in self.model_tensors:
-            raise ValueError("the checkpoint has no score.weight classification head")
-
-        self.gguf_writer.add_pooling_type(gguf.PoolingType.LAST)
-        self.gguf_writer.add_embedding_length_out(len(self.CLASS_LABELS))
-        self.gguf_writer.add_classifier_output_labels(self.CLASS_LABELS)
-        self.gguf_writer.add_decision_type(gguf.DecisionType.MINICPM_OCR_JEV)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "score.weight":
-            if data_torch.shape[0] != len(self.CLASS_LABELS):
-                raise ValueError(f"score.weight must have {len(self.CLASS_LABELS)} rows, got {data_torch.shape[0]}")
-            yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.CLS_OUT] + ".weight", data_torch)
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
 def _is_openjev_checkpoint(dir_model: Path) -> bool:
     return (dir_model / "helper" / "shim.py").is_file() and (dir_model / "config.json").is_file()
 

conversion/qwen3vl.py

@@ -144,16 +144,6 @@ class Qwen3VLVisionModel(MmprojModel):
         yield from MmprojModel.modify_tensors(self, data_torch, name, bid)
 
 
-@ModelBase.register("MiniCPMOCRForSequenceClassification")
-class MiniCPMOCRSeqClsVisionModel(Qwen3VLVisionModel):
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        # trained with the image budget of its processor, wider than the mtmd default of Qwen-VL
-        size = self.preprocessor_config["size"]
-        self.gguf_writer.add_vision_min_pixels(int(size["shortest_edge"]))
-        self.gguf_writer.add_vision_max_pixels(int(size["longest_edge"]))
-
-
 @ModelBase.register("Qwen3OmniMoeForConditionalGeneration")
 @ModelBase.example("Qwen/Qwen3-Omni-30B-A3B-Instruct")
 class Qwen3OmniMmprojModel(Qwen3VLVisionModel, Qwen25AudioModel):

conversion/qwen4exp.py

@@ -41,8 +41,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self._ple_shards: dict[int, str] = {}
         self._ple_row_dim: int | None = None
         self._mtp_fc: dict[str, Tensor] = {}
-        # halves of the merged weights, keyed by (bid, merged tensor), see _merge_shared_input
-        self._merge_parts: dict[tuple[int, gguf.MODEL_TENSOR], dict[int, Tensor]] = {}
 
     @classmethod
     def filter_tensors(cls, item):
@@ -133,79 +131,44 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         # int64 hash constants must stay exact; 1-D tensors force F32, so use KV
         if name.endswith("ple_embedding.layer_multipliers"):
             self._ple_multipliers = [int(x) for x in data_torch.tolist()]
-            return
+            return []
         if name.endswith("ple_embedding.ngram_heads_offsets"):
             self._ple_head_offsets = [int(x) for x in data_torch.tolist()]
-            return
+            return []
         if name.endswith("ple_embedding.ngram_heads_vocab_sizes"):
             self._ple_head_vocab_sizes = [int(x) for x in data_torch.tolist()]
-            return
+            return []
 
         if ".ngram_embedding.shard_" in name:
-            yield from self._place_ple_shard(data_torch, name)
-            return
+            return self._place_ple_shard(data_torch, name)
 
         # eh_proj([e ; h_s]) = fc_embedding(e) + fc_hidden(h_s) for every hc stream s
         if name.endswith((".nextn_fc_embedding.weight", ".nextn_fc_hidden.weight")):
             self._mtp_fc[name.rsplit(".", 2)[1]] = data_torch
             if len(self._mtp_fc) < 2:
-                return
+                return []
             eh = torch.cat([self._mtp_fc.pop("nextn_fc_embedding"), self._mtp_fc.pop("nextn_fc_hidden")], dim=1)
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.NEXTN_EH_PROJ, bid, ".weight"), eh)
-            return
+            return [(self.format_tensor_name(gguf.MODEL_TENSOR.NEXTN_EH_PROJ, bid, ".weight"), eh)]
 
         # one projection feeds indexer q and k; split it, as minimax-m3 does
         if ".indexer.index_qk_proj.weight" in name:
             n_q = self.hparams["indexer_n_heads"] * self.hparams["indexer_head_dim"]
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), data_torch[:n_q])
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), data_torch[n_q:])
-            return
+            q = data_torch[:n_q]
+            k = data_torch[n_q:]
+            return [
+                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_PROJ, bid, ".weight"), q),
+                (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_K_PROJ, bid, ".weight"), k),
+            ]
 
         # Gemma zero-centred gammas the inherited norm.weight rule misses
         if name.endswith((".ple.norm_key.weight", ".ple.norm_query.weight", ".ple.norm_conv.weight",
                           ".indexer.q_layernorm.weight", ".indexer.k_layernorm.weight")):
-            yield (self.map_tensor_name(name), data_torch + 1)
-            return
+            return [(self.map_tensor_name(name), data_torch + 1)]
 
         if name.endswith(".ple.conv1d.weight"):
-            yield (self.map_tensor_name(name), data_torch.squeeze())
-            return
+            return [(self.map_tensor_name(name), data_torch.squeeze())]
 
-        for new_name, data in super().modify_tensors(data_torch, name, bid):
-            yield from self._merge_shared_input(new_name, data, bid)
-
-    # matmuls that read the same input, written as one weight [a | b] so the runtime does one matmul:
-    # (merged tensor, (a, b), linear-attention layers only)
-    _SHARED_INPUT_MERGES = (
-        (gguf.MODEL_TENSOR.HC_ATTN_DOWN_INJECT, (gguf.MODEL_TENSOR.HC_ATTN_DOWN, gguf.MODEL_TENSOR.HC_ATTN_INJECT), False),
-        (gguf.MODEL_TENSOR.HC_FFN_DOWN_INJECT,  (gguf.MODEL_TENSOR.HC_FFN_DOWN,  gguf.MODEL_TENSOR.HC_FFN_INJECT),  False),
-        (gguf.MODEL_TENSOR.ATTN_QKV_GATE,       (gguf.MODEL_TENSOR.ATTN_QKV,     gguf.MODEL_TENSOR.ATTN_GATE),      True),
-        (gguf.MODEL_TENSOR.SSM_BETA_ALPHA_CAT,  (gguf.MODEL_TENSOR.SSM_BETA,     gguf.MODEL_TENSOR.SSM_ALPHA),      True),
-    )
-
-    # caches one half of a merged weight until the other half arrives, then yields [a | b]
-    def _merge_shared_input(self, new_name: str, data: Tensor, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # a pre-quantized checkpoint carries per-tensor scales, which one merged weight cannot hold
-        if bid is None or self.hparams.get("quantization_config"):
-            yield (new_name, data)
-            return
-
-        layer_types = self.hparams.get("layer_types") or []
-        is_linear = bid < len(layer_types) and layer_types[bid] == "linear_attention"
-
-        match = next(((merged, parts.index(part)) for merged, parts, linear_only in self._SHARED_INPUT_MERGES
-                      for part in parts
-                      if (is_linear or not linear_only) and self.match_model_tensor_name(new_name, part, bid)), None)
-        if match is None:
-            yield (new_name, data)
-            return
-
-        merged, idx = match
-        halves = self._merge_parts.setdefault((bid, merged), {})
-        halves[idx] = data
-        if len(halves) == 2:
-            del self._merge_parts[(bid, merged)]
-            yield (self.format_tensor_name(merged, bid), torch.cat([halves[0], halves[1]], dim=0))
+        return super().modify_tensors(data_torch, name, bid)
 
     # the shards concatenate into a tensor of well over 100 GB
     # use LazyChunkedTensor here, a single shard resident at a time
@@ -251,8 +214,6 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         super().prepare_tensors()
         if self._mtp_fc:
             raise ValueError(f"MTP projection missing its other half: {sorted(self._mtp_fc)}")
-        if self._merge_parts:
-            raise ValueError(f"merged weights missing a half: {sorted((bid, t.name) for bid, t in self._merge_parts)}")
         n_parts = self.hparams.get("split_ngram_parts", 0)
         if self._ple_shards and len(self._ple_shards) != n_parts:
             raise ValueError(

conversion/ricursive.py

@@ -1,92 +0,0 @@
-from __future__ import annotations
-
-import math
-
-from typing import Iterable, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, TextModel, gguf, logger
-
-
-@ModelBase.register("RicursiveForCausalLM")
-@ModelBase.example("ricursiveAI/Ripple-65B-Preview")
-class RicursiveModel(TextModel):
-    model_arch = gguf.MODEL_ARCH.RICURSIVE
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hparams = self.hparams
-
-        if (act := hparams.get("hidden_act", "silu")) != "silu":
-            raise ValueError(f"Ricursive: unsupported hidden_act {act!r}")
-
-        # RicursiveConfig derives the layer pattern from layer_types alone, by default full attention iff i % 5 == 4
-        if (layer_types := hparams.get("layer_types")) is None:
-            layer_types = ["full_attention" if i % 5 == 4 else "sliding_attention" for i in range(self.block_count)]
-        if len(layer_types) != self.block_count:
-            raise ValueError(f"Ricursive: layer_types has {len(layer_types)} entries, expected {self.block_count}")
-        if unknown := set(layer_types) - {"sliding_attention", "full_attention"}:
-            raise ValueError(f"Ricursive: unsupported layer_types {sorted(unknown)}")
-        self.is_swa = [t == "sliding_attention" for t in layer_types]
-        # only sliding layers use RoPE, an all-full-attention model would have no positional encoding
-        if not any(self.is_swa):
-            raise ValueError("Ricursive: layer_types has no sliding_attention layer, all-full-attention models are not supported")
-        # RicursiveConfig pops a stored hybrid_layer_pattern and recomputes it from layer_types
-        if (pattern := hparams.get("hybrid_layer_pattern")) is not None and list(pattern) != [int(swa) for swa in self.is_swa]:
-            logger.warning("Ricursive: hybrid_layer_pattern does not match layer_types, ignoring it")
-
-        if not hparams.get("gate_full_precision", True):
-            logger.warning("Ricursive: gate_full_precision is false, llama.cpp computes the attention gate in f32")
-
-        # RicursiveConfig defaults; the reference ignores the tensors of a disabled feature
-        self.use_valres = hparams.get("use_valres", False)
-        self.use_gate = hparams.get("use_gated_attention", True)
-        names = list(self.model_tensors)
-        if bias := [name for name in names if name.endswith("bias")]:
-            raise ValueError(f"Ricursive: bias tensors are not supported, found {bias[0]!r}")
-        for enabled, suffix in ((self.use_valres, ".self_attn.value_res_lambda"), (self.use_gate, ".self_attn.g_proj.weight")):
-            if enabled and sum(name.endswith(suffix) for name in names) != self.block_count:
-                raise ValueError(f"Ricursive: expected one {suffix!r} tensor per layer")
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-        # eos_token_id is [<|im_end|>, <|endoftext|>]: register the turn end as EOT so generation stops on it
-        eos = self.hparams.get("eos_token_id")
-        if isinstance(eos, list):
-            extra = [e for e in eos if e != self.hparams.get("bos_token_id")]
-            if extra:
-                self.gguf_writer.add_eot_token_id(extra[0])
-                logger.info(f"gguf: eot_token_id = {extra[0]}")
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        hparams = self.hparams
-        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
-        self.gguf_writer.add_rope_dimension_count(hparams["head_dim"])
-
-        self.gguf_writer.add_sliding_window(hparams["sliding_window"])
-        self.gguf_writer.add_sliding_window_pattern(self.is_swa)
-        # sliding layers use RoPE, full attention layers use no positional encoding
-        self.gguf_writer.add_rope_pattern(self.is_swa)
-
-        if (embed_scale := hparams.get("embed_scale")) is not None:
-            if embed_scale == 0:
-                raise ValueError(f"Ricursive: unsupported embed_scale {embed_scale!r}")
-            self.gguf_writer.add_embedding_scale(float(embed_scale))
-        # absent and 0 both mean no clamp (the reference tests `if clamp_threshold:`)
-        if (clamp := hparams.get("ffn_clamp_threshold")) is not None:
-            if clamp < 0 or not math.isfinite(clamp):
-                raise ValueError(f"Ricursive: unsupported ffn_clamp_threshold {clamp!r}")
-            self.gguf_writer.add_swiglu_clamp([float(clamp)] * self.block_count)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # value_res_lambda has no .weight suffix in the checkpoint
-        if name.endswith(".value_res_lambda"):
-            if not self.use_valres:
-                return
-            name += ".weight"
-        elif name.endswith(".g_proj.weight") and not self.use_gate:
-            return
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/sensenova_u1.py

@@ -1,114 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-import torch
-
-from .base import MmprojModel, ModelBase, gguf
-from .qwen import Qwen2Model
-
-
-@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT")
-@ModelBase.register("NEOChatModel")
-class SenseNovaU1Model(Qwen2Model):
-    model_arch = gguf.MODEL_ARCH.SENSENOVA_U1
-
-    @classmethod
-    def filter_tensors(cls, item):
-        name, _ = item
-        if not name.startswith("language_model.") or "_mot_gen" in name:
-            return None
-        return super().filter_tensors(item)
-
-    def set_gguf_parameters(self):
-        if int(self.hparams.get("num_experts") or 0) > 1:
-            raise ValueError("SenseNova U1 conversion currently supports the dense understanding branch")
-        if self.rope_parameters.get("rope_type", "default") not in ("default", None):
-            raise ValueError("SenseNova U1 scaled RoPE is not supported")
-        if self.hparams.get("attention_bias", False):
-            raise ValueError("SenseNova U1 attention bias is not supported")
-        if self.hparams.get("use_sliding_window", False):
-            raise ValueError("SenseNova U1 sliding attention is not supported")
-        super().set_gguf_parameters()
-        self.gguf_writer.add_rope_freq_base_spatial(self.hparams["rope_theta_hw"])
-
-    def get_tensors(self):
-        for name, gen in self.model_tensors.items():
-            if name.endswith((".q_norm_hw.weight", ".k_norm_hw.weight")):
-                continue
-            data = gen()
-            if name.endswith((".q_norm.weight", ".k_norm.weight")):
-                spatial = self.model_tensors[name.replace("_norm.weight", "_norm_hw.weight")]()
-                # Keep one weight vector; the graph normalizes its two halves independently.
-                data = torch.cat((data, spatial), dim=0)
-            yield name, data
-
-
-@ModelBase.register("NEOVisionModel")
-@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT")
-class SenseNovaU1VisionModel(MmprojModel):
-    def __init__(self, dir_model: Path, *args, **kwargs):
-        hparams = kwargs.pop("hparams", None)
-        if hparams is None:
-            hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
-        else:
-            hparams = dict(hparams)
-
-        vision_config = dict(hparams["vision_config"])
-        # NEOVisionModel has no transformer blocks
-        # these defaults satisfy the generic mmproj metadata contract without replacing checkpoint values
-        vision_config.setdefault("image_size", 0)
-        vision_config.setdefault("intermediate_size", 0)
-        vision_config.setdefault("num_hidden_layers", 0)
-        vision_config.setdefault("num_attention_heads", 1)
-        hparams["vision_config"] = vision_config
-        super().__init__(dir_model, *args, hparams=hparams, **kwargs)
-
-        self.preprocessor_config.setdefault("image_mean", [0.485, 0.456, 0.406])
-        self.preprocessor_config.setdefault("image_std", [0.229, 0.224, 0.225])
-
-    @classmethod
-    def filter_tensors(cls, item):
-        name, gen = item
-        if not name.startswith("vision_model."):
-            return None
-        return super().filter_tensors((name, gen))
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self.gguf_writer.add_clip_projector_type("sensenova_u1")
-        vision_config = self.hparams["vision_config"]
-        # These defaults match the known U1.5 checkpoint when older configs omit fields.
-        hidden_size = int(vision_config.get("hidden_size", 1024))
-        min_pixels = int(vision_config.get("min_pixels", 65536))
-        max_pixels = int(vision_config.get("max_pixels", 16777216))
-        downsample_ratio = float(vision_config.get("downsample_ratio", 0.5))
-        if downsample_ratio <= 0:
-            raise ValueError(f"SenseNova U1 downsample_ratio must be positive, got {downsample_ratio}")
-        merge_size = round(1.0 / downsample_ratio)
-        if abs((1.0 / downsample_ratio) - merge_size) > 1e-6:
-            raise ValueError(f"SenseNova U1 downsample_ratio must have an integer reciprocal, got {downsample_ratio}")
-        norm_eps = float(vision_config.get("layer_norm_eps", vision_config.get("rms_norm_eps", 1e-6)))
-        self.gguf_writer.add_vision_head_dim(hidden_size)
-        self.gguf_writer.add_vision_attention_layernorm_eps(norm_eps)
-        self.gguf_writer.add_vision_min_pixels(min_pixels)
-        self.gguf_writer.add_vision_max_pixels(max_pixels)
-        self.gguf_writer.add_vision_spatial_merge_size(merge_size)
-
-    def modify_tensors(self, data_torch, name, bid):
-        tensor_names = {
-            "vision_model.embeddings.patch_embedding.weight": "v.patch_embd.weight",
-            "vision_model.embeddings.patch_embedding.bias":   "v.patch_embd.bias",
-            "vision_model.embeddings.dense_embedding.weight": self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0),
-            "vision_model.embeddings.dense_embedding.bias": self.format_tensor_name(
-                gguf.MODEL_TENSOR.V_MMPROJ, 0, suffix=".bias"
-            ),
-        }
-        if name not in tensor_names:
-            raise ValueError(f"Unexpected SenseNova U1 vision tensor: {name!r}")
-        return [(tensor_names[name], data_torch)]
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0):
-            return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)

conversion/soprano.py

@@ -1,116 +0,0 @@
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import Any
-
-import torch
-
-from .base import ModelBase, MmprojModel, gguf, logger
-from .qwen import Qwen3Model
-
-
-def _is_soprano_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "decoder.pth").is_file() or not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    if config.get("model_type") != "qwen3" or config.get("bos_token_id") != 3 or config.get("eos_token_id") != 3:
-        return False
-    if not (dir_model / "tokenizer.json").is_file():
-        return False
-    with open(dir_model / "tokenizer.json", encoding="utf-8") as f:
-        added = {t["id"]: t["content"] for t in json.load(f).get("added_tokens", [])}
-    return [added.get(i) for i in range(4)] == ["[UNK]", "[TEXT]", "[START]", "[STOP]"]
-
-
-@ModelBase.register_hparams_loader(_is_soprano_checkpoint)
-def _load_soprano_hparams(dir_model: Path) -> dict[str, Any]:
-    logger.info("gguf: detected Soprano checkpoint")
-    hparams = ModelBase.load_hparams(dir_model, False, guess=False)
-    # the text model stays Qwen3, the mmproj holds the decoder
-    hparams["text_config"] = {"architectures": ["SopranoTextModel"]}
-    hparams["architectures"] = ["SopranoModel"]
-    return hparams
-
-
-@ModelBase.register("SopranoTextModel")
-class SopranoTextModel(Qwen3Model):
-    model_arch = gguf.MODEL_ARCH.QWEN3
-
-    def get_vocab_base_pre(self, tokenizer) -> str:
-        return "soprano"
-
-    def set_vocab(self):
-        super().set_vocab()
-        self.gguf_writer.add_unk_token_id(0)
-
-
-@ModelBase.register("SopranoModel")
-@ModelBase.example("ekwek/Soprano-80M", "ekwek/Soprano-1.1-80M")
-class SopranoModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = False
-
-    def _load_decoder(self) -> dict[str, torch.Tensor]:
-        state = torch.load(self.dir_model / "decoder.pth", map_location="cpu", mmap=True, weights_only=True)
-        return state.get("state_dict", state)
-
-    def get_audio_config(self):
-        state = self._load_decoder()
-        n_embd, n_inp, _ = state["decoder.embed.weight"].shape
-        n_ff = state["decoder.convnext.0.pwconv1.weight"].shape[0]
-        n_layer = sum(1 for name in state if name.endswith(".pwconv1.weight"))
-        if n_inp != self.n_embd_text:
-            raise ValueError(f"Soprano decoder input dim {n_inp} does not match the backbone dim {self.n_embd_text}")
-        return {"num_hidden_layers": n_layer, "hidden_size": n_embd, "intermediate_size": n_ff}
-
-    def set_gguf_parameters(self):
-        self.gguf_writer.add_file_type(self.ftype)
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.SOPRANO)
-        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_gen_audio_embedding_length(self.hparams["hidden_size"])
-        self.gguf_writer.add_gen_audio_feed_forward_length(self.hparams["intermediate_size"])
-        self.gguf_writer.add_gen_audio_block_count(self.hparams["num_hidden_layers"])
-        self.gguf_writer.add_gen_audio_head_count(1)
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6)
-
-    def get_tensors(self):
-        yield from self._load_decoder().items()
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if "dwconv.weight" in new_name or (name == "decoder.embed.weight" and n_dims == 3):
-            return gguf.GGMLQuantizationType.F16
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def modify_tensors(self, data_torch, name, bid):
-        if name == "head.istft.window":
-            # the ISTFT in mtmd builds its own window
-            if not torch.equal(data_torch.float(), torch.hann_window(data_torch.shape[0])):
-                raise ValueError("Soprano requires a periodic Hann window")
-            return
-        if name == "decoder.embed.weight" and data_torch.shape[-1] == 1:
-            data_torch = data_torch.squeeze(-1)
-
-        T = gguf.MODEL_TENSOR
-        if name.startswith("decoder.convnext."):
-            _, _, blk, key, *suffix = name.split(".")
-            block_map = {
-                "dwconv":  T.A_GEN_WAV_UP_DWCONV,
-                "norm":    T.A_GEN_WAV_UP_NORM,
-                "pwconv1": T.A_GEN_WAV_UP_PW1,
-                "pwconv2": T.A_GEN_WAV_UP_PW2,
-                "gamma":   T.A_GEN_WAV_UP_GAMMA,
-            }
-            yield self.format_tensor_name(block_map[key], int(blk), suffix="".join("." + s for s in suffix)), data_torch
-            return
-
-        tensor_map = {
-            "decoder.embed":            T.A_GEN_WAV_INPUT,
-            "decoder.norm":             T.A_GEN_WAV_NORM,
-            "decoder.final_layer_norm": T.A_GEN_WAV_OUTPUT_NORM,
-            "head.out":                 T.A_GEN_WAV_OUTPUT,
-        }
-        key, suffix = name.rsplit(".", 1)
-        yield self.format_tensor_name(tensor_map[key], suffix="." + suffix), data_torch

conversion/xing.py

@@ -1,133 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Callable, Iterable, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, gguf
-from .deepseek import DeepseekV2Model
-
-
-@ModelBase.register("Xing4_0ForCausalLM")
-class Xing4_0Model(DeepseekV2Model):
-    """Xing4_0 : DeepSeek-V2/V3 backbone (MLA + MoE) + mHC residual mixing."""
-
-    model_arch = gguf.MODEL_ARCH.XING4_0
-
-    skip_mtp = False
-    supports_mtp_export = True
-    _n_main_layers: int | None = None
-
-    # map (prefix, kind) -> MODEL_TENSOR enum
-    _hc_tensor_map = {
-        ("hc_attn", "fn"):    gguf.MODEL_TENSOR.HC_ATTN_FN,
-        ("hc_attn", "base"):  gguf.MODEL_TENSOR.HC_ATTN_BASE,
-        ("hc_attn", "scale"): gguf.MODEL_TENSOR.HC_ATTN_SCALE,
-        ("hc_ffn",  "fn"):    gguf.MODEL_TENSOR.HC_FFN_FN,
-        ("hc_ffn",  "base"):  gguf.MODEL_TENSOR.HC_FFN_BASE,
-        ("hc_ffn",  "scale"): gguf.MODEL_TENSOR.HC_FFN_SCALE,
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        self.block_count = self.hparams["num_hidden_layers"]
-        if not self.no_mtp:
-            self.block_count += self.hparams.get("num_nextn_predict_layers", 0)
-        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
-
-    def index_tensors(self, remote_hf_model_id: str | None = None):
-        # needed by filter_tensors() before any tensor is filtered
-        type(self)._n_main_layers = self.hparams["num_hidden_layers"]
-        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        if (titem := super().filter_tensors(item)) is None:
-            return None
-        name, gen = titem
-
-        # the NextN/MTP block lives past num_hidden_layers (model.layers.40 -> blk.40)
-        assert cls._n_main_layers is not None
-        is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers
-
-        # --no-mtp: drop the appended NextN block entirely.
-        if is_mtp and cls.no_mtp:
-            return None
-        # --mtp: keep ONLY NextN-block tensors plus the shared embeddings/
-        # norm/lm_head (so the resulting GGUF carries just the draft head).
-        if cls.mtp_only and not is_mtp and name not in (
-            "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
-        ):
-            return None
-
-        return name, gen
-
-    def set_gguf_parameters(self):
-        rope_type = self.rope_parameters.get("rope_type") or self.rope_parameters.get("type")
-        if rope_type == "rope":
-            self.rope_parameters["rope_type"] = "yarn"
-            if "type" in self.rope_parameters:
-                self.rope_parameters["type"] = "yarn"
-            # base.py's YARN branch accesses this with a direct key lookup
-            self.rope_parameters.setdefault("original_max_position_embeddings",
-                                            self.hparams.get("original_max_position_embeddings", 4096))
-
-        super().set_gguf_parameters()
-        hparams = self.hparams
-
-        # NextN/MTP prediction layers
-        if not self.no_mtp and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers")) is not None:
-            self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
-
-        self.gguf_writer.add_hyper_connection_count(
-            hparams.get("hc_mult",  1))
-        self.gguf_writer.add_hyper_connection_sinkhorn_iterations(
-            hparams.get("hc_sinkhorn_iters",  20))
-        self.gguf_writer.add_hyper_connection_epsilon(
-            hparams.get("hc_eps",  1e-6))
-
-    def set_vocab(self):
-        # Xing4_0 uses a SentencePiece tokenizer (tokenizer.model + Xing4_0Tokenizer).
-        # V2's set_vocab tries GPT2/BPE first, which fails for SPM. Use the SPM path directly.
-        # Note: no add_tokenizer_pre override — the C++ SPM load path ignores
-        # tokenizer.ggml.pre entirely (pre-tokenizers only apply to BPE vocabs).
-        self._set_vocab_sentencepiece()
-
-    def prepare_metadata(self, vocab_only: bool):
-        from_dir = self.fname_out.is_dir()
-        super().prepare_metadata(vocab_only=vocab_only)
-
-        if not self.mtp_only or not from_dir:
-            return
-
-        output_type: str = self.ftype.name.partition("_")[2]
-        fname_default: str = gguf.naming_convention(
-            self.metadata.name, self.metadata.basename, self.metadata.finetune,
-            self.metadata.version, size_label=None, output_type=output_type, model_type=None)
-        self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf"
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # handle mHC tensors: model.layers.{N}.{attn_hc|ffn_hc}.{hc_fn|hc_base|hc_scale}
-        match = re.match(r"model\.layers\.(\d+)\.(attn_hc|ffn_hc)\.(.+)$", name)
-        if match:
-            layer_idx = int(match.group(1))
-            hc_type = match.group(2)  # "attn_hc" or "ffn_hc"
-            param = match.group(3)    # "hc_fn", "hc_base", "hc_scale"
-
-            if bid is None:
-                bid = layer_idx
-
-            prefix = "hc_attn" if hc_type == "attn_hc" else "hc_ffn"
-
-            kind = {"hc_fn": "fn", "hc_base": "base", "hc_scale": "scale"}.get(param)
-            if kind is None:
-                raise ValueError(f"unexpected mHC tensor: {name}")
-            tensor_enum = self._hc_tensor_map[(prefix, kind)]
-            gguf_name = self.format_tensor_name(tensor_enum, bid)
-            yield (gguf_name, data_torch)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)

convert_hf_to_gguf.py

@@ -58,7 +58,7 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--outtype", type=str, choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"], default="auto",
-        help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto to preserve supported FP8 weights and otherwise select the highest-fidelity 16-bit float type",
+        help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type",
     )
     parser.add_argument(
         "--bigendian", action="store_true",
@@ -127,7 +127,7 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--dspark", action="store_true",
-        help="Export only the DeepSeek-V4 or Mistral Large 4 DSpark draft tensors as a separate GGUF.",
+        help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",
     )
     parser.add_argument(
         "--mistral-format", action="store_true",
@@ -255,9 +255,6 @@ def main() -> None:
             assert hparams.get("vision_encoder") is not None, "This model does not support multimodal"
             from conversion.pixtral import PixtralModel
             model_class = PixtralModel
-        elif hparams.get("moe") is not None and hparams.get("dsa") is not None:
-            from conversion.mistral4 import MistralLarge4Model
-            model_class = MistralLarge4Model
         elif hparams.get("moe") is not None:
             from conversion.mistral import MistralMoeModel
             model_class = MistralMoeModel
@@ -270,21 +267,11 @@ def main() -> None:
             sys.exit(1)
 
         if args.dspark:
-            if is_mistral_format or model_architecture == "MistralLarge4ForConditionalGeneration":
-                if model_class.__name__ != "MistralLarge4Model":
-                    logger.error("--dspark is only supported for Mistral Large 4 among Mistral-format models")
-                    sys.exit(1)
-                from conversion.mistral4 import MistralLarge4DSparkModel
-                model_class = MistralLarge4DSparkModel
-            elif model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"):
-                logger.error("--dspark is only supported for DeepseekV4ForCausalLM, DeepseekV41ForCausalLM and Mistral Large 4")
+            if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":
+                logger.error("--dspark is only supported for DeepseekV4ForCausalLM")
                 sys.exit(1)
-            elif model_architecture == "DeepseekV41ForCausalLM":
-                from conversion.deepseek41 import DeepseekV41DSparkModel
-                model_class = DeepseekV41DSparkModel
-            else:
-                from conversion.deepseek import DeepseekV4DSparkModel
-                model_class = DeepseekV4DSparkModel
+            from conversion.deepseek import DeepseekV4DSparkModel
+            model_class = DeepseekV4DSparkModel
 
         if args.mtp or args.no_mtp:
             if not model_class.supports_mtp_export:

convert_hf_to_gguf_update.py

@@ -101,7 +101,6 @@ models = [
     {"name": "command-r",        "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
     {"name": "tiny_aya",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", },
     {"name": "cohere2moe",       "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", },
-    {"name": "flint",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/RecursiveSI/flint", },
     {"name": "qwen2",            "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },
     {"name": "olmo",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
     {"name": "dbrx",             "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },
@@ -178,7 +177,6 @@ pre_computed_hashes = [
     {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.5-Air", "chkhsh": "9ca2dd618e8afaf09731a7cf6e2105b373ba6a1821559f258b272fe83e6eb902"},
     {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/zai-org/GLM-4.7-Flash", "chkhsh": "cdf5f35325780597efd76153d4d1c16778f766173908894c04afc20108536267"},
     {"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", "chkhsh": "1431a23e583c97432bc230bff598d103ddb5a1f89960c8f1d1051aaa944d0b35"},
-    {"name": "domyn-small", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/domyn/Domyn-Small-v1.0", "chkhsh": "6e4d72c6e5260b90606e26a0931fa15e4b42ec6e97719d12919f2b1f93478262"},
     {"name": "hunyuan", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-A13B-Instruct", "chkhsh": "7e57df22b1fe23a7b1e1c7f3dc4e3f96d43a4eb0836d0c6bdc3436d7b2f1c664"},
     {"name": "hunyuan-dense", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hunyuan-4B-Instruct", "chkhsh": "bba3b3366b646dbdded5dbc42d59598b849371afc42f7beafa914afaa5b70aa6"},
     {"name": "hy_v4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tencent/Hy4-preview", "chkhsh": "e6ddf9c6686791c12d698d34c31ab9be1fea9af5a3d9a6909783ab382198ae1c"},
@@ -198,7 +196,6 @@ pre_computed_hashes = [
     # jina-v2-de variants
     {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
     {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
-    {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"},
     # lfm2 variants
     {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
     # hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges

docs/backend/CANN.md

@@ -161,7 +161,7 @@ lspci -n | grep -Eo '19e5:d[0-9a-f]{3}' | cut -d: -f2
 | FP16                   | Support | Support |
 | Q8_0                   | Support | Partial |
 | Q4_0                   | Support | Partial |
-| BF16                   | Support |    No   |
+| BF16                   | Support |         |
 
 > **310P note**
 > - `Q8_0`: data transform / buffer path is implemented, and `GET_ROWS` is supported, but quantized `MUL_MAT` / `MUL_MAT_ID` are not supported.
@@ -274,9 +274,7 @@ cmake --build build --config release
 
     **Notes**:
 
-      - On 910B the CANN backend supports FP16, BF16 and Q4_0/Q8_0 models. On 310P, BF16 is not
-        supported and quantized `MUL_MAT` / `MUL_MAT_ID` are not implemented, so only FP16 models
-        are supported (see [DataType Supports](#datatype-supports)).
+      - CANN backend only supports FP16/Q4_0/Q8_0 models currently.
 
 2. **Launch inference**
 
@@ -357,7 +355,3 @@ Enable ACL graph execution during the prefill stage, default is false. This opti
 ### GGML_CANN_OPERATOR_FUSION
 
 Enable operator fusion during computation, default is false. This option fuses compatible operators (e.g., ADD + RMS_NORM) to reduce overhead and improve performance.
-
-### GGML_CANN_NO_PINNED
-
-Disable pinned (host-locked) host buffers. When it is set (any value), `aclrtMallocHost` is not called: the CANN host buffer type allocates regular CPU memory instead and the backend reports no `host_buffer` capability, so host-side staging uses pageable memory. Use it when pinned host allocation fails, or when the pinned memory is needed by other processes on the host.

docs/backend/OPENCL.md

@@ -72,8 +72,6 @@ However, A6x GPUs in phones are likely not supported due to the outdated driver
 | MXFP4                  | Support                    |
 | IQ4_NL                 | Support                    |
 
-`MUL_MAT_ID` accepts an expert id of `-1` and writes zeros to that slot. `ADD_ID` adds no bias for `-1`, including in fused MoE operations.
-
 ## Model Preparation
 
 Since common quantizations are supported now, it is recommanded to download GGUF models directly from Huggingface.

docs/backend/SYCL.md

@@ -783,8 +783,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL          | ON (mandatory)                        | Enable build with SYCL code path.           |
 | GGML_SYCL_TARGET   | INTEL *(default)*                     | Set the SYCL target device type.            |
 | GGML_SYCL_DEVICE_ARCH | Optional                           | Set the SYCL device architecture. Setting the device architecture can improve the performance. See the table [--offload-arch](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/OffloadDesign.md#--offload-arch) for a list of valid architectures. |
-| GGML_SYCL_F16      | OFF *(default)* \|ON *(optional)*     | Enable FP16 build with SYCL code path. [1.](#sycl-f16-footnote) |
-| GGML_SYCL_SUPPORT_XMX | ON *(default)* \|OFF *(Optional)* | Build the code paths that use XMX (DPAS) instructions. Set OFF for AOT builds targeting iGPUs without XMX, which otherwise fail to compile. |
+| GGML_SYCL_F16      | OFF *(default)* \|ON *(optional)*     | Enable FP16 build with SYCL code path. (1.) |
 | GGML_SYCL_GRAPH    | ON *(default)* \|OFF *(Optional)*     | Enable build with [SYCL Graph extension](https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc). |
 | GGML_SYCL_DNN      | ON *(default)* \|OFF *(Optional)*     | Enable build with oneDNN.                   |
 | GGML_SYCL_HOST_MEM_FALLBACK | ON *(default)* \|OFF *(Optional)* | Allow host memory fallback when device memory is full during quantized weight reorder. Enables inference to continue at reduced speed (reading over PCIe) instead of failing. Requires Linux kernel 6.8+. |
@@ -792,7 +791,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | CMAKE_C_COMPILER   | `icx` *(Linux)*, `icx/cl` *(Windows)* | Set `icx` compiler for SYCL code path.      |
 | CMAKE_CXX_COMPILER | `icpx` *(Linux)*, `icx` *(Windows)*   | Set `icpx/icx` compiler for SYCL code path. |
 
-1. <a id="sycl-f16-footnote"></a>FP32 or FP16 have different performance impact to LLM. Recommended to test them for better prompt processing performance on your models. You need to rebuild the code after change `GGML_SYCL_F16=OFF/ON`.
+1. FP32 or FP16 have different performance impact to LLM. Recommended to test them for better prompt processing performance on your models. You need to rebuild the code after change `GGML_SYCL_F16=OFF/ON`.
 
 ### Runtime
 
@@ -803,8 +802,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_DEV2DEV_MEMCPY | 0 (default), 1, 2 | Choose the method of dev2dev memory copy.<br>Value: <br>*  0: SYCL API (default), only support dGPUs.<br>* 1: L0 API -- Better performance, only support dGPUs, found to lead to abnormal crash in some case. <br>* 2: Host Forward -- Most stable method for all cases (including iGPU + dGPU*N), but with lower performance (-2% to -5%).<br>SYCL & L0 API are easy to be impacted by Intel GPU driver issue. When you meet the garbled output or crash issues in multiple GPUs case, try with this debug flag to work around or check the issue.|
 | GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.|
 | GGML_SYCL_ENABLE_OPT | 0 or 1 (default)| Enable optimize features for Intel GPUs. (Recommended to 0 for Intel devices older than Gen 10) |
-| GGML_SYCL_ENABLE_GRAPH | 0 (default) or 1 | Enable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still in development, performance depends on model and device. [1.](#sycl-graph-footnote) |
-| GGML_SYCL_GRAPH_EVICTION_TIMEOUT | 10 (default) or positive integer | Seconds a cached SYCL graph can go unused before it is evicted. Checked on a sweep that runs every half of this timeout. |
+| GGML_SYCL_ENABLE_GRAPH | 0 (default) or 1 | Enable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. |
 | GGML_SYCL_ENABLE_HOST_PINNED_MEM | 0 or 1 (default) | Enable host pinned memory to speed up copy data from host to device. When disable it, host memory will common malloc() on CPU. Disable it when use `--load-model mlock`.|
 | GGML_SYCL_HOST_PINNED_MEM_2G | 0 (default) or 1 | Limit the max memory allocation to be no more than 2GB when enable host pinned memory. USM allocations above 2 GiB take the relaxed/large-allocation path, which serializes H2D copies with compute and prevents copy/compute overlap. It will impact the startup time. Need more test. Depend on `GGML_SYCL_ENABLE_HOST_PINNED_MEM=1`.|
 | GGML_SYCL_UPLOAD_STAGING_SLOTS | 4 (default) or non-negative integer | Number of 8 MiB pinned host slots used to stage tensor uploads (model loading), so the host copy of one slot overlaps the transfer of the previous one. Set to 0 to use the old path: a malloc'd bounce buffer and a blocking copy per tensor. |
@@ -813,7 +811,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_ENABLE_DNN | 0 or 1 (default)| Enable running computations through oneDNN and always use oneMKL. |
 | GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. |
 | GGML_SYCL_FA_ONEDNN_MAX_KV | 0 (default, disabled) or positive integer | By default (0), all sequences are handled by the oneDNN fused SDPA path, regardless of KV length; a positive value caps that length, past which sequences fall back to the native kernel. If GPU driver watchdog resets (DEVICE_LOST) occur during long-context inference, set this near the context depth where they start, e.g. 24576. |
-| GGML_SYCL_FA_DPAS | 1 (default) or 0 | Enable the XMX flash-attention kernel for a q4_0 or q8_0 KV cache (`-ctk q4_0 -ctv q4_0` or `-ctk q8_0 -ctv q8_0`) on Xe2 or newer GPUs (Arc B-series, Lunar Lake, Panther Lake, Wildcat Lake). It reads the quantized cache directly for decode and batches of up to 31 tokens, head sizes 64, 96, 128, 256 and 512, up to 32 query heads per KV head. Set to 0 to use the native SYCL flash-attention kernels. |
 | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. |
 | GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` |
 | GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. |
@@ -822,9 +819,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. |
 | GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. Unsupported types and layouts fall back to the standalone op kernels. See `ggml_sycl_can_fuse()`. |
 | GGML_SYCL_ENABLE_ESIMD | 0 or 1 (default)| Enable ESIMD kernels when available. |
-| GGML_SYCL_ENABLE_XMX_ESIMD | 0 or 1 (default) | Use the XMX (DPAS) multi-column mat-vec kernels for reordered Q2_K to Q6_K and Q8_0 weights on Xe2 GPUs. Speeds up speculative decoding and short prompt batches. |
-| GGML_SYCL_XMX_MAX_COLS | non-negative integer, 80 (default) | Most activation columns (batch size) the `GGML_SYCL_ENABLE_XMX_ESIMD` kernels take; wider batches use the other mat-mul kernels. A larger value keeps XMX for bigger batches (as repeated launches), `0` never uses it. |
-| GGML_SYCL_XMX_GLU_MAX_COLS | 0 to 16, 16 (default) | Most columns the fused gate and up XMX kernel takes; wider batches run gate and up as separate mat-vecs. 16 is the most the kernel is built for, larger values are clamped to 16 and `0` turns the fused kernel off. |
 | GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:<br>* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M<br>* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K (MoE `MUL_MAT_ID` only)<br>Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert or row lengths that are not a multiple of 256 (32 for IQ4_NL and Q8_0) use the library GEMM. |
 | GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:<br>* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16<br>* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16<br>Clear a bit to exclude a combination, or set a single bit to force one for testing. |
 | GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. |
@@ -837,29 +831,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. |
 | GGML_SYCL_USM_SYSTEM | 0 (default) or 1 | Enable experimental support for [USM system allocations](https://github.khronos.org/SYCL_Reference/iface/usm_basic_concept.html#system-allocations) for large GPU buffers. This requires enough host memory for model weights and caches, an Intel Xe2+ GPU such as BMG or newer and supported on Linux only, with CONFIG_DRM_XE_GPUSVM enabled. |
 
-1. <a id="sycl-graph-footnote"></a>SYCL Graph support (`GGML_SYCL_ENABLE_GRAPH=1`) is limited to single-GPU workloads and may also depend on the model architecture and kernels in use.
-   Performance depends on how often a recorded graph can be reused, which will vary by workload.
-   For this reason, it is recommended to measure performance both with and without graphs enabled on your workloads.
-
-   To check whether graphs are supported for your workload, enable INFO logging (`-lv 4`) and look for `enabling SYCL graphs` or `disabling SYCL graphs`.
-   Support may differ between prefill and decode, so check the status for the phases that matter to you.
-   DEBUG logging (`-lv 5`) reports why graphs are disabled.
-
-   Then benchmark with your preferred tool, such as `llama-bench`.
-   For graph activity details (including reuse), set `GGML_SYCL_DEBUG=1` and look for `[SYCL-GRAPH]` messages.
-
-### Flash-attention kernels
-
-Attention dispatches on shape and hardware.
-
-DPAS is the quantized path. q4_0 or q8_0 KV cache, decode and batches up to 31 tokens, Xe2 and newer only. It issues dpas straight on the quantized blocks and never materialises an f16 copy. vISA gets used here because joint_matrix has no s4 type.
-
-oneDNN does XMX through SDPA but converts KV to f16 first, so it ends up on prefill: 32+ tokens and 1K+ keys. MKL is the prompt processing equivalent through oneMKL GEMM, also f16.
-
-VEC/TILE are plain kernels with no XMX in them. They run when DPAS is off or the hardware doesn't support it.
-
-joint_matrix belongs to the matmul GEMMs. Attention doesn't use it.
-
 ## Compile-time Flags
 
 Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spot.

docs/backend/snapdragon/README.md

@@ -324,15 +324,8 @@ on 4 physical NPUs, or `--devices 'HTP0[0-1:0],HTP1[0-1:1]'` on 2 physical NPUs
 - `GGML_HEXAGON_NHVX=0`
   Controls the number of HVX hardware threads to use. The default is all (actual number varies depending on the hardware version).
 
-- `GGML_HEXAGON_VTCM_SIZE=0`
-  Overrides the VTCM size used by each session. Accepts `K`, `M` and `G` suffixes; a plain number is in MB.
-  The default (0) uses the size the hardware reports. Values above the hardware size will fail to start the session.
-
-- `GGML_HEXAGON_HOSTBUF=0` (default: 1, enabled)
-  Controls host buffer (`HTP*-HOST`) allocation in shared FastRPC memory to avoid host-device tensor copies.
-  CPU-assigned weights (input embeddings like `token_embd` and `per_layer_token_embd`, and tensors overridden with `-ot ...=CPU`)
-  are placed in host buffers as well. On older drivers/FW, large models can run out of IOVA space when these are mapped.
-  Pass `--no-host` to keep CPU-assigned weights in regular CPU memory (`-ot ...=CPU` alone is not sufficient).
+- `GGML_HEXAGON_HOSTBUF=1` (default: 0, disabled)
+  Enables allocating host buffers for debugging. By default, host buffers are disabled.
 
 - `GGML_HEXAGON_DMA64=0` (default: enabled on v81+)
   Disables 64-bit DMA for model weights. Set to `1` to enable it explicitly on a supported architecture.

docs/build-s390x.md

@@ -249,7 +249,7 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
 | MXFP4      | ✅           | ❓    | ❓     |
 | Q5_0       | ✅           | ❓    | ❓     |
 | Q5_1       | ✅           | ❓    | ❓     |
-| Q8_0       | ✅           | ✅    | ❓     |
+| Q8_0       | ✅           | ❓    | ❓     |
 | Q2_K       | 🚫           | ❓    | ❓     |
 | Q3_K       | ✅           | ❓    | ❓     |
 | Q4_K       | ✅           | ❓    | ❓     |
@@ -266,11 +266,11 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
 | IQ1_M      | 🚫           | ❓    | ❓     |
 | IQ4_NL     | ✅           | ❓    | ❓     |
 | IQ4_XS     | ✅           | ❓    | ❓     |
-| FP32->FP16 | ✅           | ❓    | ❓     |
+| FP32->FP16 | 🚫           | ❓    | ❓     |
 | FP16->FP32 | 🚫           | ❓    | ❓     |
 
 -   ✅ - acceleration available
 -   🚫 - acceleration unavailable, will still run using scalar implementation
 -   ❓ - acceleration unknown, please contribute if you can test it yourself
 
-Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Nov 3, 2026.
+Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Sep 8, 2026.

docs/build.md

@@ -481,7 +481,7 @@ cmake --build build --config Release
 Now you can load the model in conversation mode using `Vulkan`
 
 ```sh
-build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2
+build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 -cnv
 ```
 
 **MSYS2**

docs/ops.md

@@ -14,22 +14,22 @@ Legend:
 
 | Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
 |-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|
-|                              ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                              ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                            CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                             CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                       CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
+|                       CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ |
 |                          CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
-|                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
@@ -38,97 +38,96 @@ Legend:
 |                              CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |               CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |          CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                     DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                     DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                      DSV4_HC_PRE | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                              DUP | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              DUP | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                              ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                   FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
-|                            FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                      GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                      GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                       GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
 |                    GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
 |                       GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                           IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                              LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ |
+|                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
-|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
+|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 |
 |                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
-|                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                     MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                     MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                     MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                     MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
+|                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                   OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                     OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                         OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
 |                              PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
 |                   PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                          POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                          POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                          POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                          POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                            REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                           REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                      REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                         RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                             ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                             ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                        RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                            SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
 |                         SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
-|                              SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
 |                              SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
-|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
 |                              SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
-|                             STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
+|                             STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                              SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |                         SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                           SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                     SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                           SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                     SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                       SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |               TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
-|                       TOP_K_MASK | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-|                              TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                              TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                            XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |

docs/ops/WebGPU.csv

@@ -12278,23 +12278,13 @@
 "WebGPU: WebGPU","SUM","type=f32,ne=[11,5,6,3],permute=[0,2,1,3]","support","1","yes","WebGPU"
 "WebGPU: WebGPU","SUM","type=f32,ne=[11,5,6,3],permute=[0,3,2,1]","support","1","yes","WebGPU"
 "WebGPU: WebGPU","SUM","type=f32,ne=[11,5,6,3],permute=[0,1,3,2]","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[10,5,4,3],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[33,1,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[33,256,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[32769,1,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[32,1,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[32,256,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[32768,1,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[11,5,6,3],permute=1,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[11,5,6,3],permute=0,slice=1","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[11,5,6,3],permute=1,slice=1","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[1,7,3,2],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[1023,5,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[1024,5,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[1025,5,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[17,65537,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[512,1,1,1],permute=0,slice=0","support","1","yes","WebGPU"
-"WebGPU: WebGPU","MEAN","type=f32,ne=[281,1536,1,1],permute=0,slice=0","support","1","yes","WebGPU"
+"WebGPU: WebGPU","MEAN","type=f32,ne=[10,5,4,3]","support","0","no","WebGPU"
+"WebGPU: WebGPU","MEAN","type=f32,ne=[33,1,1,1]","support","0","no","WebGPU"
+"WebGPU: WebGPU","MEAN","type=f32,ne=[33,256,1,1]","support","0","no","WebGPU"
+"WebGPU: WebGPU","MEAN","type=f32,ne=[32769,1,1,1]","support","0","no","WebGPU"
+"WebGPU: WebGPU","MEAN","type=f32,ne=[32,1,1,1]","support","0","no","WebGPU"
+"WebGPU: WebGPU","MEAN","type=f32,ne=[32,256,1,1]","support","0","no","WebGPU"
+"WebGPU: WebGPU","MEAN","type=f32,ne=[32768,1,1,1]","support","0","no","WebGPU"
 "WebGPU: WebGPU","SUM","type=f32,ne=[33,1,1,1]","support","1","yes","WebGPU"
 "WebGPU: WebGPU","SUM","type=f32,ne=[33,1024,1,1]","support","1","yes","WebGPU"
 "WebGPU: WebGPU","SUM","type=f32,ne=[33,256,1,1]","support","1","yes","WebGPU"
@@ -12336,16 +12326,8 @@
 "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
 "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
 "WebGPU: WebGPU","ROLL","shift0=3,shift1=-2,shift3=1,shift4=-1","support","0","no","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=3.000000,stop=3.500000,step=1.000000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-10.000000,stop=10.000000,step=0.300000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=62.750000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.000000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.250000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=254.750000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.000000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.250000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","1","yes","WebGPU"
+"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","0","no","WebGPU"
+"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","0","no","WebGPU"
 "WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","0","no","WebGPU"
 "WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU"
 "WebGPU: WebGPU","CUMSUM","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU"

docs/speculative.md

@@ -100,9 +100,8 @@ llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DSpark.gguf \
 
 `--spec-draft-n-max` is clamped to the draft model's trained block size.
 
-`--spec-draft-p-min P` truncates each drafted block at the first position whose predicted
-acceptance (from the draft's confidence head) falls below `P` (default 0 = disabled).
-Values above 0 require a draft with a confidence head.
+`--spec-draft-conf-min P` truncates each drafted block at the first position whose predicted
+acceptance (from the draft's confidence head, if present) falls below `P` (default 0 = disabled).
 
 Currently only drafts with a Qwen3 backbone are supported; support for other backbones
 (e.g. Gemma4) is planned.