ggml : remove public ggml_backend_buffer_copy_tensor()

.devops/intel.Dockerfile

@@ -64,7 +64,7 @@ RUN mkdir -p /app/full \
     && cp requirements.txt /app/full \
     && cp .devops/tools.sh /app/full/tools.sh
 
-FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base
+FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base
 
 ARG BUILD_DATE=N/A
 ARG APP_VERSION=N/A
@@ -79,8 +79,6 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
       org.opencontainers.image.url=$IMAGE_URL \
       org.opencontainers.image.source=$IMAGE_SOURCE
 
-COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl
-
 #Following versions are for multiple GPUs, since 26.x has known issue:
 #   https://github.com/ggml-org/llama.cpp/issues/21747,
 #   https://github.com/intel/compute-runtime/issues/921.

.github/workflows/behavior-check.yml

@@ -13,8 +13,6 @@ on:
       '**/*.hpp',
       '**/*.c',
       '**/*.cpp',
-      '**/*.py',
-      '**/requirements*.txt',
     ]
 
   pull_request:
@@ -26,9 +24,7 @@ on:
       '**/*.h',
       '**/*.hpp',
       '**/*.c',
-      '**/*.cpp',
-      '**/*.py',
-      '**/requirements*.txt',
+      '**/*.cpp'
     ]
 
 cache-mode: none
@@ -73,7 +69,7 @@ jobs:
         run: |
           python3 -m venv .venv
           source .venv/bin/activate
-          pip install -r requirements/requirements-convert_hf_to_gguf.txt
+          pip install jinja2==3.1.6
 
       - name: ccache-buckets-restore
         uses: ./.github/actions/ccache-buckets
@@ -138,7 +134,7 @@ jobs:
         run: |
           python3 -m venv .venv
           source .venv/Scripts/activate
-          pip install -r requirements/requirements-convert_hf_to_gguf.txt
+          pip install jinja2==3.1.6
 
       - name: Build
         id: cmake_build
@@ -160,13 +156,3 @@ jobs:
           source .venv/Scripts/activate
           cd build
           ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900
-
-      - name: ccache-clear
-        uses: ./.github/actions/ccache-clear
-        env:
-          GH_TOKEN: ${{ github.token }}
-        with:
-          key: behavior-check-windows-2025
-          older: 5m
-          min: 1
-          dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}

.github/workflows/build-apple-daily.yml

@@ -1,160 +0,0 @@
-name: CI (apple - daily)
-
-on:
-  workflow_dispatch: # allows manual triggering
-  schedule:
-    - cron: '0 0 * * *'
-
-cache-mode: none
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  GGML_NLOOP: 3
-  GGML_N_THREADS: 1
-  LLAMA_ARG_LOG_COLORS: 1
-  LLAMA_ARG_LOG_PREFIX: 1
-  LLAMA_ARG_LOG_TIMESTAMPS: 1
-
-jobs:
-  macos-latest-ios-xcode:
-    runs-on: macos-latest
-
-    permissions:
-      actions: write
-      contents: read
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-
-      - name: Setup Xcode
-        uses: ggml-org/setup-xcode@v1
-        with:
-          xcode-version: latest-stable
-
-      - name: Build
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=iOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-
-      - name: xcodebuild for swift package
-        id: xcodebuild
-        run: |
-          ./build-xcframework.sh
-
-      - name: Upload xcframework artifact
-        uses: actions/upload-artifact@v6
-        with:
-          name: llama-xcframework
-          path: build-apple/llama.xcframework/
-          retention-days: 1
-
-      - name: Build Xcode project
-        run: |
-          xcodebuild -downloadPlatform iOS
-          xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
-
-  macos-latest-tvos:
-    runs-on: macos-latest
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Build
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_BUILD_COMMON=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=tvOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-
-  macos-latest-visionos:
-    runs-on: macos-latest
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Build
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_BUILD_COMMON=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_SYSTEM_NAME=visionOS \
-            -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
-            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
-
-  macos-latest-swift:
-    runs-on: macos-latest
-    needs: macos-latest-ios-xcode
-
-    permissions:
-      actions: read
-      contents: read
-
-    strategy:
-      matrix:
-        destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-
-      - name: Download xcframework artifact
-        uses: actions/download-artifact@v7
-        with:
-          name: llama-xcframework
-          path: build-apple/llama.xcframework/
-
-      - name: Build llama.cpp with CMake
-        id: cmake_build
-        run: |
-          sysctl -a
-          cmake -B build -G Xcode \
-            -DGGML_METAL_EMBED_LIBRARY=ON \
-            -DLLAMA_OPENSSL=OFF \
-            -DLLAMA_BUILD_APP=OFF \
-            -DLLAMA_BUILD_EXAMPLES=OFF \
-            -DLLAMA_BUILD_TOOLS=OFF \
-            -DLLAMA_BUILD_TESTS=OFF \
-            -DLLAMA_BUILD_SERVER=OFF \
-            -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
-          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

.github/workflows/build-apple.yml

@@ -107,3 +107,140 @@ jobs:
           # ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
           ctest -L main -E "test-llama-archs|test-llama-context|test-speculative-rejection-sampling" --verbose --timeout 900
 
+  macos-latest-ios-xcode:
+    runs-on: macos-latest
+
+    permissions:
+      actions: write
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Xcode
+        uses: ggml-org/setup-xcode@v1
+        with:
+          xcode-version: latest-stable
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_OPENSSL=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_SYSTEM_NAME=iOS \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+      - name: xcodebuild for swift package
+        id: xcodebuild
+        run: |
+          ./build-xcframework.sh
+
+      - name: Upload xcframework artifact
+        uses: actions/upload-artifact@v6
+        with:
+          name: llama-xcframework
+          path: build-apple/llama.xcframework/
+          retention-days: 1
+
+      - name: Build Xcode project
+        run: |
+          xcodebuild -downloadPlatform iOS
+          xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
+
+  macos-latest-tvos:
+    runs-on: macos-latest
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_BUILD_COMMON=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_SYSTEM_NAME=tvOS \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+  macos-latest-visionos:
+    runs-on: macos-latest
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Build
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_BUILD_COMMON=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_SYSTEM_NAME=visionOS \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
+            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
+
+  macos-latest-swift:
+    runs-on: macos-latest
+    needs: macos-latest-ios-xcode
+
+    permissions:
+      actions: read
+      contents: read
+
+    strategy:
+      matrix:
+        destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v6
+
+      - name: Download xcframework artifact
+        uses: actions/download-artifact@v7
+        with:
+          name: llama-xcframework
+          path: build-apple/llama.xcframework/
+
+      - name: Build llama.cpp with CMake
+        id: cmake_build
+        run: |
+          sysctl -a
+          cmake -B build -G Xcode \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DLLAMA_OPENSSL=OFF \
+            -DLLAMA_BUILD_APP=OFF \
+            -DLLAMA_BUILD_EXAMPLES=OFF \
+            -DLLAMA_BUILD_TOOLS=OFF \
+            -DLLAMA_BUILD_TESTS=OFF \
+            -DLLAMA_BUILD_SERVER=OFF \
+            -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
+          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)

.github/workflows/build-ibm.yml

@@ -43,9 +43,9 @@ env:
   LLAMA_ARG_LOG_TIMESTAMPS: 1
 
 jobs:
-  ubuntu-26-s390x:
-    name: ubuntu-26-s390x (VXE ${{ matrix.vxe }})
-    runs-on: ubuntu-26.04-s390x
+  ubuntu-24-s390x:
+    name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
+    runs-on: ubuntu-24.04-s390x
 
     strategy:
       fail-fast: false
@@ -66,6 +66,12 @@ jobs:
             libjpeg-dev build-essential libssl-dev \
             git-lfs
 
+      - name: Toolchain workaround (GCC 14)
+        run: |
+          sudo apt-get install -y gcc-14 g++-14
+          echo "CC=gcc-14" >> "$GITHUB_ENV"
+          echo "CXX=g++-14" >> "$GITHUB_ENV"
+
       - name: Python Dependencies
         id: python_depends
         run: |
@@ -110,19 +116,30 @@ jobs:
 
   ubuntu-26-zdnn-s390x:
     name: ubuntu-26-zdnn-s390x
-    runs-on: ubuntu-26.04-s390x
+    runs-on: ubuntu-24.04-s390x
+    container: ubuntu:26.04  # required to get GCC 15.1 and binutils 2.44
+    defaults:
+      run:
+        shell: bash
 
     steps:
+      - name: Build Dependencies
+        id: build_depends
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends \
+            build-essential cmake git ca-certificates \
+            libssl-dev libzdnn-dev
+
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
 
-      - name: Build Dependencies
-        id: build_depends
+      - name: Toolchain workaround (GCC 15)
         run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential libssl-dev libzdnn-dev
+          apt-get install -y gcc-15 g++-15
+          echo "CC=gcc-15" >> "$GITHUB_ENV"
+          echo "CXX=g++-15" >> "$GITHUB_ENV"
 
       - name: Build with zDNN Backend
         id: cmake_build

.github/workflows/pr-video-demo-label.yml

@@ -1,37 +0,0 @@
-name: "PR video demo label"
-
-on:
-  pull_request_target:
-    types: [opened, edited, reopened, synchronize]
-
-cache-mode: none
-permissions:
-  pull-requests: write
-  issues: write
-
-jobs:
-  video-demo-label:
-    runs-on: ubuntu-slim
-    steps:
-      - name: Add or remove the video demo label
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_URL: ${{ github.event.pull_request.html_url }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          # passed through env, so the body is only matched, never run, and is safe to read from a fork
-          BODY: ${{ github.event.pull_request.body }}
-          HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
-        run: |
-          # sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment
-          # markers, or a preview linking to a demo video uploaded for this PR
-          has_demo=false
-          if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
-            grep -qE "/resolve/videos/pr/$PR_NUMBER/[0-9a-f]{10}(-[0-9]+)?\.mp4" <<< "$BODY"; then
-            has_demo=true
-          fi
-
-          if [ "$has_demo" = true ] && [ "$HAS_LABEL" = false ]; then
-            gh pr edit "$PR_URL" --add-label 'video demo'
-          elif [ "$has_demo" = false ] && [ "$HAS_LABEL" = true ]; then
-            gh pr edit "$PR_URL" --remove-label 'video demo'
-          fi

.github/workflows/release-publish.yml

@@ -172,8 +172,6 @@ jobs:
             - [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
             - [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
             - [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
-            - [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz)
-            - [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz)
             - [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
             - [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
             - [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)

.github/workflows/release.yml

@@ -180,8 +180,6 @@ jobs:
             os: ubuntu-24.04-arm
           - build: 's390x'
             os: ubuntu-24.04-s390x
-          - build: '26.04-s390x'
-            os: ubuntu-26.04-s390x
 
     runs-on: ${{ matrix.os }}
 
@@ -202,7 +200,7 @@ jobs:
         id: depends
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential libssl-dev
+          sudo apt-get install build-essential libssl-dev
 
       - name: Toolchain workaround (GCC 14)
         if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
@@ -212,7 +210,7 @@ jobs:
           echo "CXX=g++-14" >> "$GITHUB_ENV"
 
       - name: ccache
-        if: ${{ !contains(matrix.os, 's390x') }}
+        if: ${{ matrix.build != 's390x' }}
         uses: ggml-org/ccache-action@v1.2.24
         with:
           key: release-${{ matrix.os }}-cpu
@@ -244,7 +242,7 @@ jobs:
           archive: false
 
       - name: ccache-clear
-        if: ${{ !contains(matrix.os, 's390x') }}
+        if: ${{ matrix.build != 's390x' }}
         uses: ./.github/actions/ccache-clear
         with:
           key: release-${{ matrix.os }}-cpu
@@ -1628,60 +1626,6 @@ jobs:
         with:
           key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
 
-  ubuntu-26-zdnn:
-    needs: [check-release, ui-build]
-    if: ${{ needs.check-release.outputs.should_release == 'true' }}
-
-    runs-on: ubuntu-26.04-s390x
-
-    steps:
-      - name: Clone
-        id: checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Download UI build
-        uses: actions/download-artifact@v8
-        with:
-          name: llama-ui.zip
-          path: tools/ui/dist
-
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential libssl-dev libzdnn-dev
-
-      - name: Build
-        id: cmake_build
-        run: |
-          cmake -B build \
-            -DCMAKE_INSTALL_RPATH='$ORIGIN' \
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-            -DGGML_BACKEND_DL=ON \
-            -DGGML_NATIVE=OFF \
-            -DGGML_VXE=ON \
-            -DGGML_ZDNN=ON \
-            -DCMAKE_C_FLAGS="-march=arch15" \
-            -DCMAKE_CXX_FLAGS="-march=arch15" \
-            -DLLAMA_FATAL_WARNINGS=ON \
-            ${{ env.CMAKE_ARGS }}
-          cmake --build build --config Release -j $(nproc)
-
-      - name: Pack artifacts
-        id: pack_artifacts
-        run: |
-          cp LICENSE ./build/bin/
-          tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v7
-        with:
-          path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz
-          archive: false
-
   ios-xcode:
     needs: [check-release]
     if: ${{ needs.check-release.outputs.should_release == 'true' }}

CMakeLists.txt

@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)
 
 ### llama.cpp version
 set(LLAMA_VERSION_MAJOR 0)
-set(LLAMA_VERSION_MINOR 9)
+set(LLAMA_VERSION_MINOR 7)
 set(LLAMA_VERSION_PATCH 0)
 set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
 
@@ -150,7 +150,6 @@ option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured
 # Required for relocatable CMake package
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/win-app.cmake)
 
 if (NOT DEFINED LLAMA_BUILD_NUMBER)
     set(LLAMA_BUILD_NUMBER        ${BUILD_NUMBER})
@@ -270,8 +269,6 @@ if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
     add_subdirectory(tools/mtmd)
 endif()
 
-llama_apply_app_manifests()
-
 #
 # install
 #

CODEOWNERS

@@ -2,29 +2,20 @@
 # multiple collaborators per item can be specified
 #
 # ggml-org/ci               : CISC, danbev, ggerganov, netrunnereve, ngxson, taronaeo
-# ggml-org/ggml-base        : aendk, fairydreaming, ggerganov
 # ggml-org/ggml-cann        : hipudding
-# ggml-org/ggml-cpu         : angt, chaxu01, ggerganov, max-krasnyansky, taronaeo
-# ggml-org/ggml-cuda        : am17an, gaugarg-nv, IMbackK, JohannesGaessler, ORippler, thelittlefireman, ynankani
-# ggml-org/ggml-et          : marty1885
+# ggml-org/ggml-cuda        : JohannesGaessler, am17an, IMbackK, ORippler
 # ggml-org/ggml-hexagon     : lhez, max-krasnyansky
-# ggml-org/ggml-hip         : IMbackK, thelittlefireman
-# ggml-org/ggml-metal       : forforever73, ggerganov, jhen0409, masterFoad
+# ggml-org/ggml-metal       : ggerganov
 # ggml-org/ggml-opencl      : lhez, max-krasnyansky
-# ggml-org/ggml-openvino    : ravi9, wine99
-# ggml-org/ggml-riscv       : xctan
-# ggml-org/ggml-rpc         : am17an, rgerganov
-# ggml-org/ggml-sycl        : arthw, Titaniumtown
+# ggml-org/ggml-rpc         : rgerganov
+# ggml-org/ggml-sycl        : arthw
 # ggml-org/ggml-vulkan      : 0cc4m, jeffbolznv
 # ggml-org/ggml-webgpu      : reeselevine, yomaytk
-# ggml-org/ggml-zdnn        : AlekseiNikiforovIBM, Andreas-Krebbel, taronaeo
-# ggml-org/llama-base       : am17an, CISC, ggerganov, ngxson, sanmai, ServeurpersoCom
-# ggml-org/llama-common     : aldehir, angt, bartowski1182, danbev, ggerganov, ngxson, nikwen, pwilkin, ruixiang63, sanmai
-# ggml-org/llama-examples   : ggerganov
-# ggml-org/llama-models     : CISC, danbev, ggerganov, ngxson
+# ggml-org/ggml-zdnn        : taronaeo
+# ggml-org/llama-common     : ggerganov, aldehir, angt, danbev, ngxson, pwilkin
 # ggml-org/llama-mtmd       : ngxson
-# ggml-org/llama-server     : allozaur, angt, ggerganov, ngxson, ServeurpersoCom
-# ggml-org/llama-ui         : allozaur
+# ggml-org/llama-server     : ggerganov, ngxson, allozaur, angt, ServeurpersoCom
+# ggml-org/llama-ui           : allozaur
 
 /.devops/*.Dockerfile                   @ngxson
 /.github/actions/                       @ggml-org/ci
@@ -38,12 +29,28 @@
 /conversion/                            @CISC
 /convert_*.py                           @CISC
 /docs/backend/snapdragon/               @ggml-org/ggml-hexagon
-/examples/                              @ggml-org/llama-examples
+/examples/batched.swift/                @ggerganov
+/examples/batched/                      @ggerganov
+/examples/convert-llama2c-to-ggml/      @ggerganov
 /examples/debug/                        @danbev @pwilkin
+/examples/deprecation-warning/          @ggerganov
 /examples/diffusion/                    @am17an
+/examples/embedding/                    @ggerganov
+/examples/eval-callback/                @ggerganov
+/examples/export-docs/                  @ggerganov
+/examples/gen-docs/                     @ggerganov
+/examples/gguf/                         @ggerganov
 /examples/llama.android/                @ggerganov @hanyin-arm @naco-siren
+/examples/llama.swiftui/                @ggerganov
+/examples/llama.vim                     @ggerganov
+/examples/lookahead/                    @ggerganov
 /examples/lookup/                       @JohannesGaessler
 /examples/model-conversion/             @danbev
+/examples/parallel/                     @ggerganov
+/examples/passkey/                      @ggerganov
+/examples/retrieval/                    @ggerganov
+/examples/speculative-simple/           @ggerganov
+/examples/speculative/                  @ggerganov
 /ggml/cmake/                            @ggerganov
 /ggml/include/                          @ggerganov
 /ggml/src/ggml-backend-meta.cpp         @JohannesGaessler
@@ -55,7 +62,7 @@
 /ggml/src/ggml-cuda/                    @ggml-org/ggml-cuda
 /ggml/src/ggml-cuda/vendors/hip.h       @IMbackK
 /ggml/src/ggml-hexagon/                 @ggml-org/ggml-hexagon
-/ggml/src/ggml-hip/                     @ggml-org/ggml-hip
+/ggml/src/ggml-hip/                     @IMbackK
 /ggml/src/ggml-et/                      @marty1885
 /ggml/src/ggml-impl.h                   @ggerganov
 /ggml/src/ggml-metal/                   @ggml-org/ggml-metal
@@ -69,7 +76,7 @@
 /ggml/src/ggml-virtgpu/                 @kpouget
 /ggml/src/ggml-vulkan/                  @ggml-org/ggml-vulkan
 /ggml/src/ggml-webgpu/                  @ggml-org/ggml-webgpu
-/ggml/src/ggml-zdnn/                    @ggml-org/ggml-zdnn
+/ggml/src/ggml-zdnn/                    @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
 /ggml/src/ggml-zendnn/                  @avinashcpandey @Jiten1parmar
 /ggml/src/ggml.c                        @ggerganov
 /ggml/src/ggml.cpp                      @ggerganov
@@ -80,14 +87,14 @@
 /scripts/get*                           @ggerganov
 /scripts/sync*                          @ggerganov
 /scripts/snapdragon/                    @ggml-org/ggml-hexagon
-/src/                                   @ggml-org/llama-base
+/src/                                   @ggerganov
 /src/llama-adapter.*                    @CISC
 /src/llama-arch.*                       @CISC
 /src/llama-chat.*                       @ngxson
 /src/llama-graph.*                      @CISC
 /src/llama-model.*                      @CISC
 /src/llama-vocab.*                      @CISC
-/src/models/                            @ggml-org/llama-models
+/src/models/                            @CISC
 /tests/                                 @ggerganov
 /tests/test-chat.*                      @pwilkin
 /tests/test-server-component.cpp        @ggml-org/llama-server

README.md

@@ -13,7 +13,7 @@
 [![Docker](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/docker.yml?label=Docker)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
 [![Winget](https://img.shields.io/github/actions/workflow/status/ggml-org/llama.cpp/winget.yml?label=Winget)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
 
-[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
+[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
 
 </div>
 
@@ -79,7 +79,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
 | --- | --- |
 | [BLAS](docs/build.md#blas-build) | All |
 | [BLIS](docs/backend/BLIS.md) | All |
-| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
+| [CANN](docs/build.md#cann) | Ascend NPU |
 | [CUDA](docs/build.md#cuda) | Nvidia GPU |
 | [HIP](docs/build.md#hip) | AMD GPU |
 | [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |

cmake/win-app.cmake

@@ -1,66 +0,0 @@
-set(LLAMA_WIN_APP_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/../common/win-app.manifest")
-
-if(WIN32)
-    if(MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
-        set(LLAMA_WIN_APP_MSVC_ABI TRUE)
-    else()
-        set(LLAMA_WIN_APP_MSVC_ABI FALSE)
-    endif()
-
-    if(NOT LLAMA_WIN_APP_MSVC_ABI)
-        if(CMAKE_VERSION VERSION_LESS 3.21)
-            message(FATAL_ERROR "CMake 3.21 or newer is required to build for Windows with the GNU ABI")
-        endif()
-        enable_language(RC)
-        set(LLAMA_WIN_APP_MANIFEST_RC "${CMAKE_CURRENT_BINARY_DIR}/win-app-manifest.rc")
-        configure_file("${CMAKE_CURRENT_LIST_DIR}/win-app.rc.in" "${LLAMA_WIN_APP_MANIFEST_RC}" @ONLY)
-        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
-    endif()
-endif()
-
-function(llama_add_app_manifest target)
-    if(NOT WIN32)
-        return()
-    endif()
-    get_target_property(_aliased ${target} ALIASED_TARGET)
-    if(_aliased)
-        return()
-    endif()
-    get_target_property(_type ${target} TYPE)
-    if(NOT _type STREQUAL "EXECUTABLE")
-        return()
-    endif()
-    if(LLAMA_WIN_APP_MSVC_ABI)
-        target_sources(${target} PRIVATE "${LLAMA_WIN_APP_MANIFEST}")
-    else()
-        if(NOT TARGET llama-win-app-manifest-res)
-            add_library(llama-win-app-manifest-res OBJECT ${LLAMA_WIN_APP_MANIFEST_RC})
-            set_source_files_properties(${LLAMA_WIN_APP_MANIFEST_RC} PROPERTIES OBJECT_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
-        endif()
-        target_link_libraries(${target} PRIVATE "$<TARGET_OBJECTS:llama-win-app-manifest-res>")
-    endif()
-endfunction()
-
-function(llama_apply_app_manifests_dir dir)
-    get_property(_targets DIRECTORY "${dir}" PROPERTY BUILDSYSTEM_TARGETS)
-    foreach(_target IN LISTS _targets)
-        llama_add_app_manifest(${_target})
-    endforeach()
-    get_property(_subdirs DIRECTORY "${dir}" PROPERTY SUBDIRECTORIES)
-    foreach(_subdir IN LISTS _subdirs)
-        llama_apply_app_manifests_dir("${_subdir}")
-    endforeach()
-endfunction()
-
-function(llama_apply_app_manifests)
-    if(NOT WIN32)
-        return()
-    endif()
-    get_property(_subdirs DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY SUBDIRECTORIES)
-    foreach(_dir IN LISTS _subdirs)
-        get_filename_component(_name "${_dir}" NAME)
-        if(_name STREQUAL "app" OR _name STREQUAL "examples" OR _name STREQUAL "pocs" OR _name STREQUAL "tests" OR _name STREQUAL "tools")
-            llama_apply_app_manifests_dir("${_dir}")
-        endif()
-    endforeach()
-endfunction()

cmake/win-app.rc.in

@@ -1,3 +0,0 @@
-// generated from cmake/win-app.rc.in - do not edit
-// RT_MANIFEST (24), ID 1: the application manifest shared by all llama.cpp executables
-1 24 "@LLAMA_WIN_APP_MANIFEST@"

common/arg.cpp

@@ -12,6 +12,16 @@
 #include "speculative.h"
 #include "preset.h"
 
+// fix problem with std::min and std::max
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#ifndef NOMINMAX
+#   define NOMINMAX
+#endif
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
 #include <algorithm>
 #include <cinttypes>
 #include <climits>
@@ -28,6 +38,20 @@
 #include <thread> // for hardware_concurrency
 #include <vector>
 
+#ifndef __EMSCRIPTEN__
+#ifdef __linux__
+#include <linux/limits.h>
+#elif defined(_WIN32)
+#   if !defined(PATH_MAX)
+#   define PATH_MAX MAX_PATH
+#   endif
+#elif defined(_AIX)
+#include <sys/limits.h>
+#else
+#include <sys/syslimits.h>
+#endif
+#endif
+
 #define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
 
 using json = common_json;
@@ -327,7 +351,7 @@ static bool parse_bool_value(const std::string & value) {
 static std::string get_default_local_path(const std::string & url) {
     auto f = string_split<std::string>(url, '#').front();
     f = string_split<std::string>(f, '?').front();
-    return fs_get_cache_file(string_split<std::string>(f, '/').back()).string();
+    return fs_path_to_utf8(fs_get_cache_file(string_split<std::string>(f, '/').back()));
 }
 
 static bool spec_types_is_default(const common_params & params) {
@@ -710,7 +734,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
     std::vector<std::filesystem::path> paths;
 
 #if defined(_WIN32)
-    const std::filesystem::path program_data = common_get_env("PROGRAMDATA");
+    const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA");
     if (!program_data.empty()) {
         paths.push_back(program_data / "llama.cpp" / "config.ini");
     }
@@ -738,7 +762,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam
     common_preset_context ctx(ex);
     ctx.ignore_unknown_keys = true; // the same config file is shared by all programs
     for (const auto & path : found) {
-        LOG_INF("using config file: %s\n", path.string().c_str());
+        LOG_INF("using config file: %s\n", fs_path_to_utf8(path).c_str());
         common_preset global;
         common_presets presets = ctx.load_from_ini(path, global);
         global.apply_to_params(params);
@@ -1235,7 +1259,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map<com
     return true;
 }
 
+#ifdef _WIN32
+struct utf8_argv {
+    std::vector<std::string> buf;
+    std::vector<char*> ptrs;
+};
+
+static utf8_argv make_utf8_argv() {
+    utf8_argv out;
+    int wargc = 0;
+    LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
+    if (!wargv) return out;
+
+    out.buf.reserve(wargc);
+    for (int i = 0; i < wargc; ++i) {
+        int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr);
+        if (n <= 0) { out.buf.emplace_back(); continue; }
+        auto& s = out.buf.emplace_back();
+        s.resize(static_cast<size_t>(n - 1));
+        (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr);
+    }
+    LocalFree(wargv);
+
+    out.ptrs.reserve(out.buf.size() + 1);
+    for (auto& s : out.buf) out.ptrs.push_back(s.data());
+    out.ptrs.push_back(nullptr);
+    return out;
+}
+#endif
+
 bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) {
+#ifdef _WIN32
+    auto utf8 = make_utf8_argv();
+    // repair argv only when it matches the process command line
+    if (static_cast<int>(utf8.buf.size()) == argc) {
+        argv = utf8.ptrs.data();
+    }
+#endif
+
     auto ctx_arg = common_params_parser_init(params, ex, print_usage);
     const common_params params_org = ctx_arg.params; // the example can modify the default params
 
@@ -4451,14 +4512,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.tts_lang = value;
         }
     ).set_examples({LLAMA_EXAMPLE_TTS}));
-    add_opt(common_arg(
-        {"--tts-voice"}, "VOICE",
-        "voice for audio generation (model-dependent)\n"
-        "see tts/README.md for per-model usage notes",
-        [](common_params & params, const std::string & value) {
-            params.tts_voice = value;
-        }
-    ).set_examples({LLAMA_EXAMPLE_TTS}));
     add_opt(common_arg(
         {"--tts-speaker-file"}, "FNAME",
         "speaker file path for audio generation",

common/chat.cpp

@@ -190,17 +190,10 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
         } else {
             auto & parts = jmsg["content"] = json::array();
             for (const auto & part : content_parts) {
-                if (part.type == "tool_reference") {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"name", part.name},
-                    });
-                } else {
-                    parts.push_back({
-                        {"type", part.type},
-                        {"text", part.text},
-                    });
-                }
+                parts.push_back({
+                    {"type", part.type},
+                    {"text", part.text},
+                });
             }
         }
     } else {
@@ -316,7 +309,6 @@ struct common_chat_templates {
     bool has_explicit_template;  // Model had builtin template or template overridden was specified.
     std::unique_ptr<common_chat_template> template_default;  // always set (defaults to chatml)
     std::unique_ptr<common_chat_template> template_tool_use;
-    std::unique_ptr<common_chat_template> template_tts;  // the prompt of a text to speak
 };
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -378,16 +370,12 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
                             throw std::invalid_argument("Missing content part type: " + part.dump());
                         }
                         const auto & type = part.at("type");
-                        common_chat_msg_content_part msg_part;
-                        msg_part.type = type;
-                        if (type == "text" || type == "media_marker") {
-                            msg_part.text = part.at("text");
-                        } else if (type == "tool_reference") {
-                            // names a request tool, the template renders its definition
-                            msg_part.name = part.at("name");
-                        } else {
+                        if (type != "text" && type != "media_marker") {
                             throw std::invalid_argument("Unsupported content part type: " + type.dump());
                         }
+                        common_chat_msg_content_part msg_part;
+                        msg_part.type = type;
+                        msg_part.text = part.at("text");
                         msg.content_parts.push_back(msg_part);
                     }
                 } else if (!content.is_null()) {
@@ -622,17 +610,13 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
 
     auto result = json::array();
     for (const auto & tool : tools) {
-        json function {
-            { "name",        tool.name },
-            { "description", tool.description },
-            { "parameters",  json::parse(tool.parameters) },
-        };
-        if (tool.defer_loading) {
-            function["defer_loading"] = true;
-        }
         result.push_back({
             { "type",     "function" },
-            { "function", function },
+            { "function", {
+                { "name", tool.name },
+                { "description", tool.description },
+                { "parameters", json::parse(tool.parameters) },
+            }},
         });
     }
     return result;
@@ -669,19 +653,10 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
                 }
 
                 const auto & function = tool.at("function");
-
-                // "defer_loading" sits on the function, or on the tool next to "type"; the function wins
-                const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
-                                                                        : tool.value("defer_loading", json(false));
-                if (!defer.is_boolean()) {
-                    throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
-                }
-
                 result.push_back({
                     /* .name = */ function.at("name"),
                     /* .description = */ function.value("description", ""),
                     /* .parameters = */ function.value("parameters", json::object()).dump(),
-                    /* .defer_loading = */ defer.get<bool>(),
                 });
             }
         }
@@ -804,70 +779,19 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
     return tmpls->has_explicit_template;
 }
 
-// renders a task template (e.g. "tts") with the given variables
-static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
-    jinja::context ctx(tmpl.source());
-    jinja::global_from_json(ctx, vars, false);
-    ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
-    ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
-    jinja::runtime runtime(ctx);
-    return jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
-}
-
-// LFM3 format detection: template renders deferred tools inside <additional_tools>...</additional_tools>
-static bool is_lfm3_template(const std::string & src) {
-    return src.find("<additional_tools>") != std::string::npos;
-}
-
-// LFM3-Audio: a known language code maps to its name, no language lets the model detect it
-static std::string lfm3_asr_system_prompt(const std::string & language) {
-    static const std::map<std::string, std::string> names = {
-        { "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
-        { "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
-        { "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
-        { "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
-    };
-    if (language.empty()) {
-        return "Perform ASR.";
-    }
-    std::string code = language;
-    std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); });
-    const auto it = names.find(code);
-    return "Perform ASR in " + (it != names.end() ? it->second : language) + ".";
-}
-
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {
     common_chat_prompt_preset asr_preset;
     asr_preset.system = "";
     asr_preset.user   = "Transcribe audio to text";
 
-    if (chat_templates && chat_templates->template_default) {
-        const std::string src = chat_templates->template_default->source();
-        if (is_lfm2_template(src)) {
-            asr_preset.system = "Perform ASR.";
-            asr_preset.user   = "";
-        } else if (is_lfm3_template(src)) {
-            asr_preset.system = lfm3_asr_system_prompt(language);
-            asr_preset.user   = "";
-            asr_preset.has_language = true;
-        }
+    if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {
+        asr_preset.system = "Perform ASR.";
+        asr_preset.user   = "";
     }
 
     return asr_preset;
 }
 
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language, const std::string & voice) {
-    if (!chat_templates || !chat_templates->template_tts) {
-        return "";
-    }
-    return common_chat_template_render(*chat_templates->template_tts, {
-        {"messages", json::array({{{"role", "user"}, {"content", text}}})},
-        {"language", language},
-        {"voice",    voice},
-    });
-}
-
 std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
     if (!variant.empty()) {
         if (variant == "tool_use") {
@@ -978,16 +902,6 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
             LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());
         }
     }
-    // task templates of the model, kept with a chat template override
-    if (model) {
-        if (const char * src = llama_model_chat_template(model, "tts")) {
-            try {
-                tmpls->template_tts = std::make_unique<common_chat_template>(src, token_bos, token_eos);
-            } catch (const std::exception & e) {
-                LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
-            }
-        }
-    }
     return tmpls;
 }
 
@@ -1803,9 +1717,6 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena &          src_pars
     //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
 
     common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
-    if (!is_partial) {
-        flags |= COMMON_PEG_PARSE_FLAG_COMPLETE;
-    }
     if (params.debug) {
         flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
     }
@@ -1943,13 +1854,12 @@ const common_chat_msg & common_chat_session::finish(const common_chat_input & ch
     return result;
 }
 
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
     GGML_ASSERT(chat_templates != nullptr);
     GGML_ASSERT(chat_templates->template_default != nullptr);
-    // take the more expressive template when available
-    const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps
-                                                                      : chat_templates->template_default->caps;
-    common_json result = caps.to_map();
-    result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
-    return result;
+    if (chat_templates->template_tool_use != nullptr) {
+        // take the more expressive template when available
+        return chat_templates->template_tool_use->caps.to_map();
+    }
+    return chat_templates->template_default->caps.to_map();
 }

common/chat.h

@@ -45,7 +45,6 @@ struct common_chat_tool_call {
 struct common_chat_msg_content_part {
     std::string type;
     std::string text;
-    std::string name; // tool_reference: name of a tool from the request tools
 
     // TODO @ngxson : no known chat templates support reasoning_content in content parts yet
     //                this can be useful for models with interleaved thinking (like Kimi-K2)
@@ -53,7 +52,7 @@ struct common_chat_msg_content_part {
     // std::string reasoning_content;
 
     bool operator==(const common_chat_msg_content_part & other) const {
-        return type == other.type && text == other.text && name == other.name;
+        return type == other.type && text == other.text;
     }
 };
 
@@ -227,9 +226,6 @@ struct common_chat_tool {
     std::string name;
     std::string description;
     std::string parameters;
-
-    // withheld from the system prompt, a tool_reference reveals it later in the conversation
-    bool defer_loading = false;
 };
 
 enum common_chat_tool_choice {
@@ -472,7 +468,7 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_to
 common_json common_chat_tool_parameters(const common_json & function);
 
 // get template caps, useful for reporting to server /props endpoint
-common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);
+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
 
 common_chat_prompt common_chat_template_direct_apply(
     const common_chat_template & tmpl,
@@ -491,12 +487,6 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
 struct common_chat_prompt_preset {
     std::string system;
     std::string user;
-    bool has_language = false; // the preset already names the requested language
 };
 
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
-
-// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
-// language and voice are optional, the template defines their defaults
-std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
-                                       const std::string & language = "", const std::string & voice = "");
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);

common/common.cpp

@@ -219,23 +219,6 @@ int32_t common_cpu_get_num_math() {
             }
         }
     }
-#elif defined(__riscv) && defined(__linux__)
-    std::ifstream cpuinfo("/proc/cpuinfo");
-    std::string line;
-    int32_t n_a100 = 0;
-    while (std::getline(cpuinfo, line)) {
-        std::istringstream fields(line);
-        std::string key;
-        std::string separator;
-        uint64_t marchid;
-        if (fields >> key >> separator >> std::hex >> marchid &&
-            key == "marchid" && separator == ":" && marchid == UINT64_C(0x8000000041000002)) {
-            ++n_a100;
-        }
-    }
-    if (n_a100 > 0) {
-        return n_a100;
-    }
 #elif defined(__powerpc64__) || defined(__powerpc__)
     int32_t smt_factor = 1;
     int phy_cpus = common_cpu_get_num_physical_cores();
@@ -934,6 +917,12 @@ std::string wstring_to_utf8(const std::wstring & str) {
 }
 #endif
 
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path) {
+    const auto value = path.u8string();
+    return std::string(value.begin(), value.end());
+}
+
 void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
     // a random suffix gives every writer its own temporary file, across processes and containers
     std::random_device rd;
@@ -957,7 +946,7 @@ void fs_write_atomic(const std::filesystem::path & path, const std::string & dat
 
     if (file.fail() || ec) {
         std::filesystem::remove(path_tmp, ec);
-        throw std::runtime_error("failed to write file: " + path.string());
+        throw std::runtime_error("failed to write file: " + fs_path_to_utf8(path));
     }
 }
 
@@ -983,9 +972,20 @@ void common_set_env(const std::string & name, const std::string & value) {
 #endif
 }
 
+std::filesystem::path common_get_path_from_env(const std::string & name) {
+#if defined(_WIN32)
+    const std::wstring wname = utf8_to_wstring(name);
+    const wchar_t * wvalue = _wgetenv(wname.c_str());
+    return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path();
+#else
+    const char * value = std::getenv(name.c_str());
+    return value ? std::filesystem::path(value) : std::filesystem::path();
+#endif
+}
+
 #if !defined(_WIN32)
 static std::filesystem::path get_home_directory() {
-    std::filesystem::path home = common_get_env("HOME");
+    std::filesystem::path home = common_get_path_from_env("HOME");
     if (!home.empty()) {
         return home;
     }
@@ -998,19 +998,19 @@ static std::filesystem::path get_home_directory() {
 #endif
 
 std::filesystem::path fs_get_cache_directory() {
-    std::filesystem::path cache_directory = common_get_env("LLAMA_CACHE");
+    std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE");
     if (!cache_directory.empty()) {
         return cache_directory;
     }
 #if defined(_WIN32)
-    cache_directory = common_get_env("LOCALAPPDATA");
+    cache_directory = common_get_path_from_env("LOCALAPPDATA");
     if (cache_directory.empty()) {
         throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
     }
 #elif defined(__APPLE__)
     cache_directory = get_home_directory() / "Library/Caches";
 #else
-    cache_directory = common_get_env("XDG_CACHE_HOME");
+    cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
     if (cache_directory.empty()) {
         cache_directory = get_home_directory() / ".cache";
     }
@@ -1021,12 +1021,12 @@ std::filesystem::path fs_get_cache_directory() {
 std::filesystem::path fs_get_config_directory() {
     std::filesystem::path config_directory;
 #if defined(_WIN32)
-    config_directory = common_get_env("APPDATA");
+    config_directory = common_get_path_from_env("APPDATA");
     if (config_directory.empty()) {
         throw std::runtime_error("Failed to find %APPDATA% directory");
     }
 #else
-    config_directory = common_get_env("XDG_CONFIG_HOME");
+    config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
     if (config_directory.empty()) {
         config_directory = get_home_directory() / ".config";
     }
@@ -1040,7 +1040,7 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) {
     std::error_code ec;
     common_create_directories(cache_directory, ec);
     if (ec) {
-        throw std::runtime_error("failed to create cache directory: " + cache_directory.string());
+        throw std::runtime_error("failed to create cache directory: " + fs_path_to_utf8(cache_directory));
     }
     return cache_directory / std::filesystem::u8path(filename);
 }
@@ -1188,8 +1188,6 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     { COMMON_DECISION_TYPE_H2O_LIGHTNING,  "h2o-lightning" },
     { COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
-    { COMMON_DECISION_TYPE_LFM3_D1,        "lfm3-d1"       },
-    { COMMON_DECISION_TYPE_LFM3_D1_OMNI,   "lfm3-d1-omni"  },
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 
@@ -1473,7 +1471,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
         if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
         if (params.control_vector_layer_end   <= 0) params.control_vector_layer_end   = llama_model_n_layer(model);
 
-        const auto cvec = common_control_vector_load(params.control_vectors, llama_model_n_layer(model));
+        const auto cvec = common_control_vector_load(params.control_vectors);
         if (cvec.n_embd == -1) {
             return res;
         }
@@ -1995,7 +1993,7 @@ float common_embd_similarity_cos(const float * embd1, const float * embd2, int n
 // Control vector utils
 //
 
-static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info, int32_t n_layer) {
+static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info) {
     common_control_vector_data result = { -1, {} };
 
     ggml_context * ctx = nullptr;
@@ -2036,10 +2034,6 @@ static common_control_vector_data common_control_vector_load_one(const common_co
             COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str());
             result.n_embd = -1;
             break;
-        } else if (layer_idx > n_layer) {
-            COM_ERR("direction tensor layer index %d exceeds the model layer count %d in %s\n", layer_idx, n_layer, load_info.fname.c_str());
-            result.n_embd = -1;
-            break;
         }
 
         struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());
@@ -2063,10 +2057,10 @@ static common_control_vector_data common_control_vector_load_one(const common_co
         }
 
         // extend if necessary - do not store data for layer 0 (it's not used)
-        result.data.resize(std::max(result.data.size(), (size_t) result.n_embd * layer_idx), 0.0f);
+        result.data.resize(std::max(result.data.size(), static_cast<size_t>(result.n_embd * layer_idx)), 0.0f);
 
         const float * src = (const float *) tensor->data;
-        float * dst = result.data.data() + (size_t) result.n_embd * (layer_idx - 1);  // layer 1 at [0]
+        float * dst = result.data.data() + result.n_embd * (layer_idx - 1);  // layer 1 at [0]
         for (int j = 0; j < result.n_embd; j++) {
             dst[j] += src[j] * load_info.strength;  // allows multiple directions for same layer in same file
         }
@@ -2084,11 +2078,11 @@ static common_control_vector_data common_control_vector_load_one(const common_co
     return result;
 }
 
-common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer) {
+common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos) {
     common_control_vector_data result = { -1, {} };
 
     for (const auto & info : load_infos) {
-        auto cur = common_control_vector_load_one(info, n_layer);
+        auto cur = common_control_vector_load_one(info);
 
         if (cur.n_embd == -1) {
             result.n_embd = -1;
@@ -2264,16 +2258,6 @@ int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq
     return size() - 1;
 }
 
-int32_t common_batch::add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output) {
-    token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} };
-    t.embd_row = row;
-    for (int32_t j = 0; j < n_pos; ++j) {
-        t.pos[j] = pos[j];
-    }
-    tokens.push_back(t);
-    return size() - 1;
-}
-
 llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
     GGML_ASSERT(batch && "common_batch was not initialized with a context");
     GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size());
@@ -2294,12 +2278,6 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
             if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) {
                 GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i);
             }
-        } else if (t.embd_row >= 0) {
-            idx = llama_batch_ext_add_embd_row(res, t.seq_id, t.embd_row);
-            if (idx < 0) {
-                GGML_ABORT("%s: failed to add embd row %d at index %d (error %d, n = %d)\n", __func__, t.embd_row, i, idx, n);
-            }
-            llama_batch_ext_set_pos(res, idx, t.pos.data());
         } else {
             idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd);
             if (idx < 0) {

common/common.h

@@ -772,7 +772,6 @@ struct common_params {
 
     // TTS params
     std::string tts_lang = "";
-    std::string tts_voice = "";
     std::string tts_speaker_file = "";
 
     bool is_gen_docs = false; // whether we are running inside llama-gen-docs
@@ -910,6 +909,9 @@ std::wstring utf8_to_wstring(const std::string & str);
 std::string  wstring_to_utf8(const std::wstring & str);
 #endif
 
+// returns the path as a UTF-8 string, preserving its separators
+std::string fs_path_to_utf8(const std::filesystem::path & path);
+
 //
 // Environment utils
 //
@@ -919,6 +921,9 @@ std::string  wstring_to_utf8(const std::wstring & str);
 std::string common_get_env(const std::string & name);
 void        common_set_env(const std::string & name, const std::string & value);
 
+// reads a path from the environment, an unset variable gives an empty path
+std::filesystem::path common_get_path_from_env(const std::string & name);
+
 //
 // Filesystem utils
 //
@@ -975,8 +980,6 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
     COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
-    COMMON_DECISION_TYPE_LFM3_D1,       // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
-    COMMON_DECISION_TYPE_LFM3_D1_OMNI,  // same as lfm3-d1
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 
@@ -1093,7 +1096,6 @@ struct common_batch {
         llama_embd   state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
         std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
         int32_t      decision_order = 0; // see llama_batch_ext_set_decision_order()
-        int32_t      embd_row = -1; // see add_embd_row(), -1 if none
     };
 
     std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
@@ -1136,10 +1138,6 @@ struct common_batch {
     // pos points to n_pos positions
     int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);
 
-    // add an entry whose embedding is row `row` of the layer input device buffer of the context's ctx_other
-    // pos points to n_pos positions
-    int32_t add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output);
-
     int32_t size() const { return (int32_t) tokens.size(); }
 };
 
@@ -1236,9 +1234,8 @@ struct common_control_vector_load_info {
 };
 
 // Load control vectors, scale each by strength, and add them together.
-// Direction layer indices must be in [1, n_layer].
 // On error, returns {-1, empty}
-common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer);
+common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos);
 
 //
 // Split utils

common/download.cpp

@@ -748,8 +748,7 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
     }
 
     for (const auto & t : tags) {
-        // match the tag as a full token, so ":F16" does not match a BF16 file
-        std::regex pattern("(^|[-./])" + t + "[.-]", std::regex::icase);
+        std::regex pattern(t + "[.-]", std::regex::icase);
         for (const auto & f : files) {
             if (gguf_filename_is_model(f.path) &&
                 std::regex_search(f.path, pattern)) {
@@ -1006,7 +1005,7 @@ std::string common_docker_resolve_model(const std::string & docker) {
         std::string model_filename = repo;
         std::replace(model_filename.begin(), model_filename.end(), '/', '_');
         model_filename += "_" + tag + ".gguf";
-        std::string local_path = fs_get_cache_file(model_filename).string();
+        std::string local_path = fs_path_to_utf8(fs_get_cache_file(model_filename));
 
         const std::string blob_url = url_prefix + "/blobs/" + gguf_digest;
         common_download_opts opts;

common/hf-cache.cpp

@@ -45,7 +45,7 @@ static fs::path get_cache_directory() {
             {HOME_DIR,                fs::path(".cache") / "huggingface" / "hub"}
         };
         for (const auto & entry : entries) {
-            if (fs::path base = common_get_env(entry.var); !base.empty()) {
+            if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) {
                 return entry.path.empty() ? base : base / entry.path;
             }
         }
@@ -63,7 +63,7 @@ static fs::path get_cache_directory() {
 }
 
 std::string get_cache_path() {
-    return get_cache_directory().string();
+    return fs_path_to_utf8(get_cache_directory());
 }
 
 static std::string folder_name_to_repo(const std::string & folder) {
@@ -340,11 +340,11 @@ hf_files get_repo_files(const std::string & repo_id,
             file.url = endpoint + repo_id + "/resolve/" + commit + "/" + file.path;
 
             fs::path final_path = commit_path / subpath;
-            file.final_path = final_path.string();
+            file.final_path = fs_path_to_utf8(final_path);
 
             if (!file.oid.empty() && !fs::exists(final_path)) {
                 fs::path local_path = blobs_path / file.oid;
-                file.local_path = local_path.string();
+                file.local_path = fs_path_to_utf8(local_path);
             } else {
                 file.local_path = file.final_path;
             }
@@ -411,7 +411,7 @@ hf_files get_cached_files(const std::string & repo_id) {
         if (!fs::exists(snapshots_path)) {
             continue;
         }
-        std::string _repo_id = folder_name_to_repo(repo.path().filename().string());
+        std::string _repo_id = folder_name_to_repo(fs_path_to_utf8(repo.path().filename()));
 
         if (!is_valid_repo_id(_repo_id)) {
             continue;
@@ -436,7 +436,7 @@ hf_files get_cached_files(const std::string & repo_id) {
                 file.repo_id = _repo_id;
                 const auto generic_path = path.generic_u8string();
                 file.path = std::string(generic_path.begin(), generic_path.end());
-                file.local_path = entry.path().string();
+                file.local_path = fs_path_to_utf8(entry.path());
                 file.final_path = file.local_path;
                 files.push_back(std::move(file));
             }
@@ -516,7 +516,7 @@ bool remove_cached_repo(const std::string & repo_id) {
     std::error_code ec;
     auto removed = fs::remove_all(repo_path, ec);
     if (ec) {
-        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str());
+        LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, fs_path_to_utf8(repo_path).c_str(), ec.message().c_str());
         return false;
     }
     return removed > 0;

common/jinja/caps.cpp

@@ -6,9 +6,7 @@
 // we can remove it in the future when we figure out a better way to define inputs using jinja::value
 #include "json.h"
 
-#include <algorithm>
 #include <functional>
-#include <optional>
 #include <sstream>
 
 #define FILENAME "jinja-caps"
@@ -186,10 +184,6 @@ caps caps_get(jinja::program & prog) {
                     {
                         {"role", "user"},
                         {"content", json::array({
-                            {
-                                {"type", "text"},
-                                {"text", content_marker}
-                            }
                         })}
                     }
                 });
@@ -590,58 +584,6 @@ caps caps_get(jinja::program & prog) {
         }
     );
 
-    if (result.supports_reasoning_effort) {
-        JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort levels");
-
-        static const std::vector<std::string> candidates = { "minimal", "low", "medium", "high", "xhigh", "max" };
-
-        // renders a single user turn with the given effort, nothing when the template raises
-        const auto render = [&](const std::string & effort) {
-            std::optional<std::string> rendered;
-            caps_try_execute(
-                prog,
-                [&]() {
-                    // messages
-                    return json::array({
-                        {
-                            {"role", "user"},
-                            {"content", "User message"}
-                        },
-                    });
-                },
-                [&](context & ctx) {
-                    ctx.set_val("enable_thinking", mk_val<value_bool>(true));
-                    caps_apply_reasoning_effort(ctx, effort);
-                },
-                nullptr, // tools_fn
-                [&](context &, bool success, value &, value &, const std::string & output) {
-                    if (success) {
-                        rendered = output;
-                    }
-                }
-            );
-            return rendered;
-        };
-
-        // an unknown level shows the render of the template fallback, a level that renders the same is not honored
-        // levels sharing a render are aliases and the highest one is kept
-        const auto unknown = render("REASONING_EFFORT_UNKNOWN");
-        std::vector<std::string> renders;
-        for (const auto & level : candidates) {
-            const auto rendered = render(level);
-            if (!rendered || rendered == unknown) {
-                continue;
-            }
-            const auto it = std::find(renders.begin(), renders.end(), *rendered);
-            if (it != renders.end()) {
-                result.reasoning_effort_levels[it - renders.begin()] = level;
-            } else {
-                renders.push_back(*rendered);
-                result.reasoning_effort_levels.push_back(level);
-            }
-        }
-    }
-
     JJ_DEBUG("%s\n", result.to_string().c_str());
 
     return result;

common/jinja/caps.h

@@ -4,7 +4,6 @@
 
 #include <string>
 #include <map>
-#include <vector>
 
 namespace jinja {
 
@@ -20,9 +19,6 @@ struct caps {
     // supports reasoning effort levels
     bool supports_reasoning_effort = false;
 
-    // reasoning effort levels the template renders distinctly, from lowest to highest
-    std::vector<std::string> reasoning_effort_levels;
-
     // one of the 2 content capabilities must be true
     bool supports_string_content = true;
     bool supports_typed_content = false;

common/jinja/runtime.cpp

@@ -307,13 +307,58 @@ value binary_expression::execute_impl(context & ctx) const {
     throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type());
 }
 
+static value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false) {
+    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
+    if (ctx.is_get_stats) {
+        value_t::stats_t::mark_used(input);
+        input->stats.ops.insert(name);
+    }
+    auto builtins = input->get_builtins();
+    auto it = builtins.find(name);
+    if (it != builtins.end()) {
+        JJ_DEBUG("Binding built-in '%s'", name.c_str());
+        return mk_val<value_func>(name, it->second, input);
+    }
+    if (undef_on_missing) {
+        return mk_val<value_undefined>(name);
+    }
+    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
+}
+
 static value apply_filter(context & ctx, const statement_ptr & filter, value input) {
     JJ_DEBUG("Applying filter to %s", input->type().c_str());
 
+    auto set_filter_alias = [](auto & filter_id) {
+        if (filter_id == "count") {
+            filter_id = "length";
+        } else if (filter_id == "d") {
+            filter_id = "default";
+        } else if (filter_id == "e") {
+            filter_id = "escape";
+        } else if (filter_id == "trim") {
+            filter_id = "strip";
+        }
+    };
+
     if (is_stmt<identifier>(filter)) {
         auto filter_id = cast_stmt<identifier>(filter)->val;
+
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str());
-        return apply_filter_by_name(ctx, filter_id, input, func_args(ctx));
+        // TODO: Refactor filters so this coercion can be done automatically
+        if (!input->is_undefined() && !is_val<value_string>(input) && (
+            filter_id == "capitalize" ||
+            filter_id == "lower" ||
+            filter_id == "replace" ||
+            filter_id == "strip" ||
+            filter_id == "title" ||
+            filter_id == "upper" ||
+            filter_id == "wordcount"
+        )) {
+            JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
+            input = mk_val<value_string>(input->as_string());
+        }
+        return try_builtin_func(ctx, filter_id, input)->invoke(func_args(ctx));
 
     } else if (is_stmt<call_expression>(filter)) {
         auto call = cast_stmt<call_expression>(filter);
@@ -322,13 +367,14 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp
         }
         auto filter_id = cast_stmt<identifier>(call->callee)->val;
 
+        set_filter_alias(filter_id);
         JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str());
         func_args args(ctx);
         for (const auto & arg_expr : call->args) {
             args.push_back(arg_expr->execute(ctx));
         }
 
-        return apply_filter_by_name(ctx, filter_id, input, args);
+        return try_builtin_func(ctx, filter_id, input)->invoke(args);
 
     } else {
         throw std::runtime_error("Invalid filter expression");

common/jinja/value.cpp

@@ -72,17 +72,6 @@ const std::vector<value> & func_args::get_args() const {
     return args;
 }
 
-void func_args::ensure_kwargs(std::initializer_list<const char *> names) const {
-    for (const auto & arg : args) {
-        if (is_val<value_kwarg>(arg)) {
-            auto * kwarg = cast_val<value_kwarg>(arg);
-            if (std::find(names.begin(), names.end(), kwarg->key) == names.end()) {
-                throw raised_exception("Function '" + func_name + "' got an unexpected keyword argument '" + kwarg->key + "'");
-            }
-        }
-    }
-}
-
 /**
  * Function that mimics Python's array slicing.
  */
@@ -145,14 +134,6 @@ static value empty_value_fn(const func_args &) {
         return mk_val<T>();
     }
 }
-
-template<typename T>
-static value empty_value_no_args_fn(const func_args & args) {
-    args.ensure_kwargs();
-    args.ensure_count(1, 1);
-    return empty_value_fn<T>(args);
-}
-
 template<typename T>
 static value test_type_fn(const func_args & args) {
     args.ensure_count(1);
@@ -613,52 +594,6 @@ const func_builtins & global_builtins() {
     return builtins;
 }
 
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) {
-    JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
-    if (ctx.is_get_stats) {
-        value_t::stats_t::mark_used(input);
-        input->stats.ops.insert(name);
-    }
-    auto builtins = input->get_builtins();
-    auto it = builtins.find(name);
-    if (it != builtins.end()) {
-        JJ_DEBUG("Binding built-in '%s'", name.c_str());
-        return mk_val<value_func>(name, it->second, input);
-    }
-    if (undef_on_missing) {
-        return mk_val<value_undefined>(name);
-    }
-    throw std::runtime_error("Unknown (built-in) filter '" + name + "' for type " + input->type());
-}
-
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args) {
-    if (filter_id == "count") {
-        filter_id = "length";
-    } else if (filter_id == "d") {
-        filter_id = "default";
-    } else if (filter_id == "e") {
-        filter_id = "escape";
-    } else if (filter_id == "trim") {
-        filter_id = "strip";
-    }
-
-    // TODO: Refactor filters so this coercion can be done automatically
-    if (!input->is_undefined() && !is_val<value_string>(input) && (
-        filter_id == "capitalize" ||
-        filter_id == "lower" ||
-        filter_id == "replace" ||
-        filter_id == "strip" ||
-        filter_id == "title" ||
-        filter_id == "upper" ||
-        filter_id == "wordcount"
-    )) {
-        JJ_DEBUG("Coercing %s to String for '%s' filter", input->type().c_str(), filter_id.c_str());
-        input = mk_val<value_string>(input->as_string());
-    }
-
-    return try_builtin_func(ctx, filter_id, input)->invoke(args);
-}
-
 
 const func_builtins & value_int_t::get_builtins() const {
     static const func_builtins builtins = {
@@ -732,22 +667,16 @@ const func_builtins & value_string_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
         {"upper", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().uppercase();
             return mk_val<value_string>(str);
         }},
         {"lower", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().lowercase();
             return mk_val<value_string>(str);
         }},
         {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
             value val_input = args.get_pos(0);
             if (!is_val<value_string>(val_input)) {
                 throw raised_exception("strip() first argument must be a string");
@@ -778,15 +707,11 @@ const func_builtins & value_string_t::get_builtins() const {
             }
         }},
         {"title", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().titlecase();
             return mk_val<value_string>(str);
         }},
         {"capitalize", [](const func_args & args) -> value {
-            args.ensure_kwargs();
-            args.ensure_count(1, 1);
             args.ensure_vals<value_string>();
             jinja::string str = args.get_pos(0)->as_string().capitalize();
             return mk_val<value_string>(str);
@@ -1088,10 +1013,6 @@ const func_builtins & value_bool_t::get_builtins() const {
     return builtins;
 }
 
-[[noreturn]] static value array_sum_not_implemented(const func_args &) {
-    throw not_implemented_exception("Array sum builtin not implemented");
-}
-
 const func_builtins & value_array_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
@@ -1158,9 +1079,7 @@ const func_builtins & value_array_t::get_builtins() const {
         {"select", select_or_reject<false, false>},
         {"rejectattr", select_or_reject<true, true>},
         {"reject", select_or_reject<true, false>},
-        {"sum", array_sum_not_implemented},
         {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
             args.ensure_count(1, 3);
             if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("join() first argument must be an array");
@@ -1198,28 +1117,13 @@ const func_builtins & value_array_t::get_builtins() const {
         {"tojson", tojson},
         {"map", [](const func_args & args) -> value {
             args.ensure_count(2);
-            value val = args.get_pos(0);
-            if (!is_val<value_array>(val)) {
+            if (!is_val<value_array>(args.get_pos(0))) {
                 throw raised_exception("map: first argument must be an array");
             }
             if (!is_val<value_kwarg>(args.get_args().at(1))) {
-                value filter = args.get_pos(1);
-                if (!is_val<value_string>(filter)) {
-                    throw raised_exception("map: filter name must be a string");
-                }
-                auto out = mk_val<value_array>();
-                func_args filter_args(args.ctx);
-                const auto & all_args = args.get_args();
-                for (size_t i = 2; i < all_args.size(); ++i) {
-                    filter_args.push_back(all_args[i]);
-                }
-                const std::string filter_id = filter->as_string().str();
-                auto arr = val->as_array();
-                for (const auto & item : arr) {
-                    out->push_back(apply_filter_by_name(args.ctx, filter_id, item, filter_args));
-                }
-                return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+                throw not_implemented_exception("map: filter-mapping not implemented");
             }
+            value val       = args.get_pos(0);
             value attribute = args.get_kwarg_or_pos("attribute", 1);
             value default_val = args.get_kwarg("default", mk_val<value_undefined>());
             auto out = mk_val<value_array>();
@@ -1228,7 +1132,7 @@ const func_builtins & value_array_t::get_builtins() const {
                 value attr_val = get_attribute(item, attribute, default_val);
                 out->push_back(attr_val);
             }
-            return is_val<value_tuple>(val) ? mk_val<value_tuple>(out->as_array()) : out;
+            return is_val<value_tuple>(val) ? mk_val<value_tuple>(std::move(out->as_array())) : out;
         }},
         {"append", [](const func_args & args) -> value {
             args.ensure_count(2);
@@ -1501,18 +1405,14 @@ const func_builtins & value_none_t::get_builtins() const {
 const func_builtins & value_undefined_t::get_builtins() const {
     static const func_builtins builtins = {
         {"default", default_value},
-        {"capitalize", empty_value_no_args_fn<value_string>},
+        {"capitalize", empty_value_fn<value_string>},
         {"first", empty_value_fn<value_undefined>},
         {"items", empty_value_fn<value_array>},
-        {"join", [](const func_args & args) -> value {
-            args.ensure_kwargs({"d", "attribute"});
-            args.ensure_count(1, 3);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"join", empty_value_fn<value_string>},
         {"last", empty_value_fn<value_undefined>},
         {"length", empty_value_fn<value_int>},
         {"list", empty_value_fn<value_array>},
-        {"lower", empty_value_no_args_fn<value_string>},
+        {"lower", empty_value_fn<value_string>},
         {"map", empty_value_fn<value_array>},
         {"max", empty_value_fn<value_undefined>},
         {"min", empty_value_fn<value_undefined>},
@@ -1525,16 +1425,12 @@ const func_builtins & value_undefined_t::get_builtins() const {
         {"selectattr", empty_value_fn<value_array>},
         {"sort", empty_value_fn<value_array>},
         {"string", empty_value_fn<value_string>},
-        {"strip", [](const func_args & args) -> value {
-            args.ensure_kwargs({"chars"});
-            args.ensure_count(1, 2);
-            return empty_value_fn<value_string>(args);
-        }},
+        {"strip", empty_value_fn<value_string>},
         {"sum", empty_value_fn<value_int>},
-        {"title", empty_value_no_args_fn<value_string>},
+        {"title", empty_value_fn<value_string>},
         {"truncate", empty_value_fn<value_string>},
         {"unique", empty_value_fn<value_array>},
-        {"upper", empty_value_no_args_fn<value_string>},
+        {"upper", empty_value_fn<value_string>},
         {"wordcount", empty_value_fn<value_int>},
     };
     return builtins;

common/jinja/value.h

@@ -7,7 +7,6 @@
 #include <cmath>
 #include <cstdint>
 #include <functional>
-#include <initializer_list>
 #include <map>
 #include <memory>
 #include <set>
@@ -177,8 +176,6 @@ protected:
 //
 
 const func_builtins & global_builtins();
-value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
-value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
 
 std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
 
@@ -661,7 +658,6 @@ public:
     size_t count() const { return args.size(); }
     void push_back(const value & val);
     void push_front(const value & val);
-    void ensure_kwargs(std::initializer_list<const char *> names = {}) const;
     void ensure_count(size_t min, size_t max = 999) const {
         size_t n = args.size();
         if (n < min || n > max) {

common/parsers/gpt-oss.cpp

@@ -71,7 +71,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
         auto content         = p.rule("message-content", p.until("<|end|>"));
         auto channel         = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
         auto constrain_type  = p.chars("[A-Za-z0-9_-]", 1, -1);
-        auto constraint      = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
 
         // Occasionally, gpt-oss-20b will prefix channels with this commentary
         auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));
@@ -85,7 +84,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
 
         auto analysis = p.ref("analysis");
         auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);
-        auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(content));
+        auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));
 
         // Consume any unsolicited tool calls, e.g. builtin functions
         auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));
@@ -93,6 +92,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
         auto any = p.rule("any", preamble | analysis);
 
         if (has_response_format) {
+            auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
             auto response_format = p.rule("response-format",
                 p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
                 p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
@@ -109,6 +109,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
                 const auto   params   = common_chat_tool_parameters(function);
 
                 auto func_name  = p.literal(" to=functions.") + p.tool_name(p.literal(name));
+                auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
                 auto args       = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params));
 
                 // recipient in role header

common/parsers/qwen3-coder.cpp

@@ -74,24 +74,15 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
     data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
         auto generation_prompt = p.literal(GEN_PREFIX);
 
-        // The template strips the newline before </think>, so leave it out of the reasoning
-        auto think_body  = p.until_one_of({ "\n</think>", "</think>", "<tool_call>" });
-        auto think_close = p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"));
-
-        auto reasoning_extracted = p.optional("<think>" + p.space() + p.reasoning(think_body) + think_close);
-        auto reasoning_skipped   = p.optional("<think>" + p.space() + think_body + think_close);
-
         auto reasoning = p.eps();
         if (supports_reasoning && extract_reasoning) {
-            reasoning = reasoning_extracted;
+            reasoning = p.optional("<think>" + p.space() +
+                                   p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
+                                   (p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
         }
 
         // Response format parser
         if (has_response_format) {
-            // The answer must be bare JSON, so the think block is consumed even when it is not extracted
-            if (supports_reasoning && !extract_reasoning) {
-                reasoning = reasoning_skipped;
-            }
             return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
         }
 
@@ -175,15 +166,8 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
             auto calls      = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
             auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
 
-            // The templates separate content from the tool calls with one or two newlines,
-            // stop the content before them
-            std::vector<std::string> content_ends;
-            for (const auto & start : tool_call_starts) {
-                content_ends.insert(content_ends.end(), { "\n\n" + start, "\n" + start, start });
-            }
-
             return generation_prompt +
-                   (reasoning << p.content(p.until_one_of(content_ends)) << tool_calls);
+                   (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
         }
 
         // Content only parser

common/peg-parser.cpp

@@ -721,9 +721,8 @@ struct parser_executor {
                 return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
             }
 
-            if (match == common_trie::PARTIAL_MATCH && !ctx.is_complete()) {
-                // Found a partial match extending to end of input, return everything before it.
-                // On a complete input the delimiter can no longer form, so the bytes belong to the text
+            if (match == common_trie::PARTIAL_MATCH) {
+                // Found a partial match extending to end of input, return everything before it
                 return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
             }
 

common/peg-parser.h

@@ -162,10 +162,9 @@ struct common_peg_parse_result {
 };
 
 enum common_peg_parse_flags {
-    COMMON_PEG_PARSE_FLAG_NONE     = 0,
-    COMMON_PEG_PARSE_FLAG_LENIENT  = 1 << 0,
-    COMMON_PEG_PARSE_FLAG_DEBUG    = 1 << 1,
-    COMMON_PEG_PARSE_FLAG_COMPLETE = 1 << 2,
+    COMMON_PEG_PARSE_FLAG_NONE    = 0,
+    COMMON_PEG_PARSE_FLAG_LENIENT = 1 << 0,
+    COMMON_PEG_PARSE_FLAG_DEBUG   = 1 << 1,
 };
 
 inline common_peg_parse_flags operator|(common_peg_parse_flags a, common_peg_parse_flags b) {
@@ -210,7 +209,6 @@ struct common_peg_parse_context {
 
     bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }
     bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }
-    bool is_complete() const { return flags & COMMON_PEG_PARSE_FLAG_COMPLETE; }
 };
 
 class common_peg_arena;

common/preset.cpp

@@ -171,12 +171,12 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     std::map<std::string, std::map<std::string, std::string>> parsed;
 
     if (!std::filesystem::exists(path)) {
-        throw std::runtime_error("preset file does not exist: " + path.string());
+        throw std::runtime_error("preset file does not exist: " + fs_path_to_utf8(path));
     }
 
     std::ifstream file(path);
     if (!file.good()) {
-        throw std::runtime_error("failed to open server preset file: " + path.string());
+        throw std::runtime_error("failed to open server preset file: " + fs_path_to_utf8(path));
     }
 
     std::string contents((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
@@ -225,7 +225,7 @@ static std::map<std::string, std::map<std::string, std::string>> parse_ini_from_
     common_peg_parse_context ctx(contents);
     const auto result = parser.parse(ctx);
     if (!result.success()) {
-        throw std::runtime_error("failed to parse server config file: " + path.string());
+        throw std::runtime_error("failed to parse server config file: " + fs_path_to_utf8(path));
     }
 
     std::string current_section = COMMON_PRESET_DEFAULT_NAME;
@@ -323,7 +323,7 @@ common_presets common_preset_context::load_from_ini(const std::filesystem::path
                 }
                 LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str());
             } else if (ignore_unknown_keys) {
-                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.string().c_str());
+                LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), fs_path_to_utf8(path).c_str());
             } else {
                 throw std::runtime_error(string_format(
                     "option '%s' not recognized in preset '%s'",
@@ -401,7 +401,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
             if (!entry.is_regular_file(ec)) {
                 continue;
             }
-            const std::string fname = entry.path().filename().string();
+            const std::string fname = fs_path_to_utf8(entry.path().filename());
             if (!string_ends_with(fname, ".gguf")) {
                 continue;
             }
@@ -422,9 +422,9 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         if (!path.empty()) {
             models.push_back({
                 /* name        */ name,
-                /* path        */ path.string(),
-                /* path_mmproj */ mmproj_file.string(), // can be empty
-                /* path_draft  */ draft_file.string()   // can be empty
+                /* path        */ fs_path_to_utf8(path),
+                /* path_mmproj */ fs_path_to_utf8(mmproj_file), // can be empty
+                /* path_draft  */ fs_path_to_utf8(draft_file)    // can be empty
             });
         }
     };
@@ -432,13 +432,13 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
     for (const auto & entry : std::filesystem::directory_iterator(dir)) {
         std::error_code ec;
         if (entry.is_directory(ec)) {
-            scan_subdir(entry.path(), entry.path().filename().string());
+            scan_subdir(entry.path(), fs_path_to_utf8(entry.path().filename()));
             continue;
         }
         if (!entry.is_regular_file(ec)) {
             continue;
         }
-        const std::string fname = entry.path().filename().string();
+        const std::string fname = fs_path_to_utf8(entry.path().filename());
         if (!string_ends_with(fname, ".gguf")) {
             continue;
         }
@@ -450,7 +450,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m
         string_replace_all(name, ".gguf", "");
         models.push_back({
             /* name        */ name,
-            /* path        */ entry.path().string(),
+            /* path        */ fs_path_to_utf8(entry.path()),
             /* path_mmproj */ "",
             /* path_draft  */ ""
         });

common/reasoning-budget.cpp

@@ -136,8 +136,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to
             // track the end sequence within forced_tokens so it is also reported on DONE
             const int32_t match = ctx->end_matcher.advance(token);
             ctx->force_pos++;
-            // an end tag that is already in the prompt (e.g. "<think></think>") also ends forcing
-            if (match >= 0 || ctx->force_pos >= ctx->forced_tokens.size()) {
+            if (ctx->force_pos >= ctx->forced_tokens.size()) {
                 ctx->state = REASONING_BUDGET_DONE;
                 ctx->end_match = match;
                 COM_TRC("%s", "forced sequence complete, done\n");

common/speculative.cpp

@@ -1027,9 +1027,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
     const int32_t * target_layer_ids   = nullptr; // model_dft's extract layer indices
     uint32_t        target_layer_ids_n = 0;
 
-    // inject target features from the target's device buffer instead of host copies (env LLAMA_DFLASH_DEV=0 disables)
-    bool use_dev_features = false;
-
     common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
             common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
         : common_speculative_impl(type, n_seq, params.draft.n_max)
@@ -1144,23 +1141,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true);
         }
 
-        // the device buffer keeps layers in ascending id order, the fc input expects target_layer_ids order
-        {
-            const char * env = getenv("LLAMA_DFLASH_DEV");
-            const bool ascending = std::adjacent_find(target_layer_ids, target_layer_ids + target_layer_ids_n,
-                    [](int32_t a, int32_t b) { return a >= b; }) == target_layer_ids + target_layer_ids_n;
-
-            use_dev_features = !is_lilicorr && ascending && (env == nullptr || atoi(env) != 0);
-            if (use_dev_features) {
-                llama_set_embeddings_layer_inp_dev(ctx_tgt, true);
-                use_dev_features = llama_supports_embd_row(ctx_dft);
-                if (!use_dev_features) {
-                    llama_set_embeddings_layer_inp_dev(ctx_tgt, false);
-                }
-            }
-            LOG_INF("%s: DFlash target features via %s\n", __func__, use_dev_features ? "device buffer" : "host copies");
-        }
-
         // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits.
         llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr);
         llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise
@@ -1253,25 +1233,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
             for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) {
                 const int32_t n_chunk = std::min(n_ubatch, n_rows - offset);
 
-                if (use_dev_features) {
-                    // the features are the target batch rows in the target's device buffer
-                    batch_inject.clear();
-                    for (int32_t i = 0; i < n_chunk; ++i) {
-                        const int32_t   row        = i_batch_beg[seq_id] + offset + i;
-                        const llama_pos p          = batch_in.tokens[row].pos[0];
-                        const llama_pos pos_arr[4] = { p, p, p, 0 };
-                        batch_inject.add_embd_row(row, pos_arr, seq_id, false);
-                    }
-
-                    const int32_t rc = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch_inject.get());
-                    if (rc != 0) {
-                        LOG_ERR("%s: llama_process(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n",
-                                __func__, rc, (int) n_chunk, (int) offset);
-                        return false;
-                    }
-                    continue;
-                }
-
                 // gather target features per extract layer; the fused decode encodes and
                 // injects them into the K/V cache at the target positions
                 features_buf.resize((size_t) n_chunk * n_embd_enc);
@@ -1613,12 +1574,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             }
         }
 
+        llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ false);
+        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true);
+
         is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt;
         chain_heads   = n_mtp_layers > 1 && !is_mem_shared;
 
-        llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ is_mem_shared);
-        llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true);
-
         if (chain_heads) {
             this->params.n_max = std::min(this->params.n_max, n_mtp_layers);
 
@@ -1772,28 +1733,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             }
 
             const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1;
-            if (is_mem_shared && !batch_in.tokens[i_batch_end[seq_id]].output) {
-                verify_h_rows[seq_id] = 0;
-                continue;
-            }
+            verify_h_rows[seq_id] = n_rows;
+            verify_h[seq_id].resize((size_t) n_rows * n_embd);
 
-            const bool all_rows_output = std::all_of(
-                    batch_in.tokens.begin() + i_batch_beg[seq_id],
-                    batch_in.tokens.begin() + i_batch_end[seq_id] + 1,
-                    [](const common_batch::token & token) { return token.output; });
-            const int32_t h_rows = is_mem_shared && !all_rows_output ? 1 : n_rows;
-            const int32_t h_offset = is_mem_shared && !all_rows_output ? i_batch_end[seq_id] : i_batch_beg[seq_id];
-
-            verify_h_rows[seq_id] = h_rows;
-            verify_h[seq_id].resize((size_t) h_rows * n_embd);
-
-            for (int32_t i = 0; i < h_rows; ++i) {
-                const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, h_offset + i);
+            for (int32_t i = 0; i < n_rows; ++i) {
+                const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, i_batch_beg[seq_id] + i);
                 std::memcpy(verify_h[seq_id].data() + (size_t) i * n_embd, h, row_bytes);
             }
 
             std::memcpy(pending_h[seq_id].data(),
-                    verify_h[seq_id].data() + (size_t) (h_rows - 1) * n_embd, row_bytes);
+                    verify_h[seq_id].data() + (size_t) (n_rows - 1) * n_embd, row_bytes);
         }
 
         return true;
@@ -1979,29 +1928,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
         const size_t row_bytes = (size_t) n_embd * sizeof(float);
         std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes);
     }
-
-    bool get_state(llama_seq_id seq_id, std::vector<uint8_t> & data) const override {
-        if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) {
-            return false;
-        }
-
-        const size_t row_bytes = (size_t) n_embd * sizeof(float);
-        data.resize(row_bytes);
-        std::memcpy(data.data(), pending_h[seq_id].data(), row_bytes);
-        return true;
-    }
-
-    void set_state(llama_seq_id seq_id, const std::vector<uint8_t> & data) override {
-        if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) {
-            return;
-        }
-
-        const size_t row_bytes = (size_t) n_embd * sizeof(float);
-        if (data.size() != row_bytes) {
-            return;
-        }
-        std::memcpy(pending_h[seq_id].data(), data.data(), row_bytes);
-    }
 };
 
 // state of self-speculation (simple implementation, not ngram-map)

common/win-app.manifest

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
-  <application xmlns="urn:schemas-microsoft-com:asm.v3">
-    <windowsSettings>
-      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
-      <longPathAware xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">true</longPathAware>
-    </windowsSettings>
-  </application>
-</assembly>

conversion/__init__.py

@@ -45,9 +45,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ChameleonForConditionalGeneration": "chameleon",
     "ChatGLMForConditionalGeneration": "chatglm",
     "ChatGLMModel": "chatglm",
-    "ChatterboxModel": "chatterbox",
     "ClefModel": "clef",
-    "CloverMoeModelLM": "clover",
     "CodeShellForCausalLM": "codeshell",
     "CogVLMForCausalLM": "cogvlm",
     "Cohere2MoeForCausalLM": "command_r",
@@ -95,7 +93,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "FalconForCausalLM": "falcon",
     "FalconH1ForCausalLM": "falcon_h1",
     "FalconMambaForCausalLM": "mamba",
-    "FishOmniModel": "drama3",
     "GPT2LMHeadModel": "gpt2",
     "GPTBigCodeForCausalLM": "starcoder",
     "GPTNeoXForCausalLM": "gptneox",
@@ -175,11 +172,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "D1OmniModel": "lfm2",
     "Lfm2Model": "lfm2",
     "Lfm2MoeForCausalLM": "lfm2",
-    "Lfm3D1Model": "lfm3",
-    "Lfm3ForCausalLM": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
-    "LimiteForCausalLM": "limite",
     "Llama4ForCausalLM": "llama",
     "Llama4ForConditionalGeneration": "llama",
     "LlamaBidirectionalModel": "llama",
@@ -199,7 +191,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "MambaLMHeadModel": "mamba",
     "MapleForCausalLM": "maple",
     "MellumForCausalLM": "mellum",
-    "Metis16ForCausalLM": "metis",
     "MiMoV2FlashForCausalLM": "mimo",
     "MiMoV2ForCausalLM": "mimo",
     "MiniCPM3ForCausalLM": "minicpm",
@@ -217,7 +208,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "Ministral3Model": "mistral3",
     "Mistral3ForConditionalGeneration": "mistral3",
     "Mistral3Model": "mistral3",
-    "MistralLarge4ForConditionalGeneration": "mistral4",
     "VoxtralRealtimeForConditionalGeneration": "mistral3",
     "MistralForCausalLM": "llama",
     "MixtralForCausalLM": "llama",
@@ -226,7 +216,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "ModernBertForSequenceClassification": "bert",
     "ModernBertModel": "bert",
     "NanbeigeForCausalLM": "nanbeige",
-    "Nemotron3_5TranscribeForConditionalGeneration": "nemotron",
     "NemotronForCausalLM": "nemotron",
     "NemotronHForCausalLM": "nemotron",
     "NemotronHPuzzleForCausalLM": "nemotron",
@@ -332,7 +321,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
 
 MMPROJ_MODEL_MAP: dict[str, str] = {
     "AudioFlamingo3ForConditionalGeneration": "ultravox",
-    "ChatterboxModel": "chatterbox",
     "ClefModel": "clef",
     "CogVLMForCausalLM": "cogvlm",
     "Cohere2VisionForConditionalGeneration": "command_r",
@@ -347,7 +335,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "DotsOCRForCausalLM": "dotsocr",
     "EmbeddingGemma2Model": "gemma",
     "Exaone4_5_ForConditionalGeneration": "exaone",
-    "FishOmniModel": "drama3",
     "Gemma3ForConditionalGeneration": "gemma",
     "Gemma3nForConditionalGeneration": "gemma",
     "Gemma4ForConditionalGeneration": "gemma",
@@ -371,9 +358,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Lfm2AudioForConditionalGeneration": "lfm2",
     "D1OmniModel": "lfm2",
     "Lfm2VlForConditionalGeneration": "lfm2",
-    "Lfm3AudioForConditionalGeneration": "lfm3",
-    "Lfm3OmniForConditionalGeneration": "lfm3",
-    "Lfm3VlForConditionalGeneration": "lfm3",
     "LightOnOCRForConditionalGeneration": "lighton_ocr",
     "Llama4ForConditionalGeneration": "llama4",
     "LlavaForConditionalGeneration": "llava",
@@ -386,7 +370,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "MiniCPMOcrForConditionalGeneration": "minicpm",
     "Mistral3ForConditionalGeneration": "llava",
     "Mistral3Model": "llava",
-    "Nemotron3_5TranscribeForConditionalGeneration": "nemotron",
     "NemotronH_Omni_Reasoning_V3": "nemotron",
     "NemotronH_Nano_VL_V2": "nemotron",
     "MuseGlimmerForConditionalGeneration": "muse_glimmer",

conversion/base.py

@@ -16,7 +16,6 @@ from pathlib import Path
 from hashlib import sha256
 from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Literal, Sequence, TypeVar, cast
 from itertools import chain
-from math import prod
 from transformers import AutoConfig
 
 import numpy as np
@@ -156,7 +155,6 @@ class ModelBase:
         self.fuse_gate_up_exps = fuse_gate_up_exps
         self._gate_exp_buffer: dict[int, Tensor] = {}
         self._up_exp_buffer: dict[int, Tensor] = {}
-        self._unfusable_gate_up_layers: set[int] = set()
         self.fuse_qkv = fuse_qkv
         self._q_buffer: dict[int, Tensor] = {}
         self._k_buffer: dict[int, Tensor] = {}
@@ -177,9 +175,7 @@ class ModelBase:
         self._nvfp4_layer_algo: dict[str, str | None] = {} # per-layer quant_algo, keyed by HF module path
         self._prec_a4: dict[str, bool] = {} # gguf tensor name -> can use 4-bit (A4) activations
         self._fp8_as_q8 = fp8_as_q8
-        self._preserve_fp8 = ftype == gguf.LlamaFileType.GUESSED and not fp8_as_q8
         self._fp8_dequantized: set[str] = set()
-        self._fp8_e4m3_preserved: set[str] = set()
 
         # Apply heuristics to figure out typical tensor encoding based on first tensor's dtype
         # NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.
@@ -225,6 +221,8 @@ class ModelBase:
         tensors: dict[str, Callable[[], Tensor]] = {}
 
         if remote_hf_model_id is not None:
+            is_safetensors = True
+
             logger.info(f"Using remote model with HuggingFace id: {remote_hf_model_id}")
             remote_tensors = gguf.utility.SafetensorRemote.get_list_tensors_hf_model(remote_hf_model_id)
             for name, remote_tensor in remote_tensors.items():
@@ -235,13 +233,9 @@ class ModelBase:
 
             return tensors
 
-        return self.index_local_tensors("model" if not self.is_mistral_format else "consolidated")
-
-    # index the local "<prefix>*.safetensors" files (or pytorch_model*.bin), through filter_tensors
-    def index_local_tensors(self, prefix: str) -> dict[str, Callable[[], Tensor]]:
-        tensors: dict[str, Callable[[], Tensor]] = {}
+        prefix = "model" if not self.is_mistral_format else "consolidated"
         part_names: list[str] = ModelBase.get_model_part_names(self.dir_model, prefix, ".safetensors")
-        is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / f"{prefix}.safetensors.index.json").is_file())
+        is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / "model.safetensors.index.json").is_file())
         if not is_safetensors:
             part_names = ModelBase.get_model_part_names(self.dir_model, "pytorch_model", ".bin")
 
@@ -249,7 +243,7 @@ class ModelBase:
         tensor_names_from_parts: set[str] = set()
 
         if not self.is_mistral_format:
-            index_name = f"{prefix}.safetensors" if is_safetensors else "pytorch_model.bin"
+            index_name = "model.safetensors" if is_safetensors else "pytorch_model.bin"
             index_name += ".index.json"
             index_file = self.dir_model / index_name
 
@@ -332,7 +326,7 @@ class ModelBase:
 
     def dequant_model(self):
         # If all quantized tensors were already handled (e.g. pure NVFP4), skip
-        if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv", ".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")) for k in self.model_tensors):
+        if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv")) for k in self.model_tensors):
             return
 
         tensors_to_remove: list[str] = []
@@ -356,12 +350,6 @@ class ModelBase:
             def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor:
                 scale = scale.float()
 
-                # ModelOpt block scales are pre-shaped for broadcasting, e.g. [out/128, 1, in/128, 1]
-                if block_size is None and scale.ndim == 2 * weight.ndim and all(n == 1 for n in scale.shape[1::2]):
-                    assert all(w % n == 0 for w, n in zip(weight.shape, scale.shape[0::2]))
-                    block_size = [w // n for w, n in zip(weight.shape, scale.shape[0::2])]
-                    scale = scale.reshape(tuple(scale.shape[0::2]))
-
                 if block_size is not None:
                     dim_offset = scale.ndim - len(block_size)
                     for i, size in enumerate(block_size):
@@ -452,7 +440,7 @@ class ModelBase:
 
                 return (unpacked * scale.unsqueeze(-1).float()).reshape(shape)
 
-            def dequant_fp8(fp8_configs: list[dict[str, Any]] | None = None) -> None:
+            def dequant_fp8() -> None:
                 for name in self.model_tensors.keys():
                     if name.endswith(".weight_scale"):
                         weight_name = name.removesuffix("_scale")
@@ -461,29 +449,14 @@ class ModelBase:
                             continue
                         w = self.model_tensors[weight_name]
                         s = self.model_tensors[name]
-                        block_size = None
                         is_fp8_weight = False
-                        if self._fp8_as_q8 or fp8_configs is not None:
-                            weight = w()
-                            is_fp8_weight = weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
-                        if fp8_configs is not None:
-                            if not is_fp8_weight:
-                                raise ValueError(f"Expected remaining weight {weight_name!r} to be FP8, got {weight.dtype}")
-                            scale = s()
-                            is_channel_scale = scale.ndim >= 1 and scale.ndim <= weight.ndim and scale.shape[0] == weight.shape[0] and all(size == 1 for size in scale.shape[1:])
-                            if prod(scale.shape) != 1 and not is_channel_scale:
-                                block_configs = [c for c in fp8_configs if c.get("strategy") == "block"]
-                                if not block_configs or any(not c.get("block_structure") or c.get("group_size") is not None for c in block_configs):
-                                    raise ValueError(f"FP8 weight {weight_name!r} with scale shape {list(scale.shape)} requires a block layout")
-                                block_sizes = {tuple(c["block_structure"]) for c in block_configs}
-                                if len(block_sizes) != 1:
-                                    raise ValueError(f"FP8 weight {weight_name!r} has ambiguous block layouts")
-                                block_size = next(iter(block_sizes))
-                        self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+                        if self._fp8_as_q8:
+                            is_fp8_weight = w().dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
+                        self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), None)
                         tensors_to_remove.append(name)
-                        if self._fp8_as_q8 and is_fp8_weight:
+                        if is_fp8_weight:
                             self._fp8_dequantized.add(weight_name)
-                    if name.endswith((".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")):
+                    if name.endswith((".input_scale", ".k_scale", ".v_scale")):
                         tensors_to_remove.append(name)
 
             if quant_method == "bitnet":
@@ -494,7 +467,7 @@ class ModelBase:
                         s = self.model_tensors[name]
                         self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s())
                         tensors_to_remove.append(name)
-            elif quant_method in ("fp8", "bsgemm-moe"):  # bsgemm-moe: drama-3, same block layout
+            elif quant_method == "fp8":
                 block_size = quant_config.get("weight_block_size")
                 for name in self.model_tensors.keys():
                     if name.endswith("_scale_inv"):
@@ -548,37 +521,29 @@ class ModelBase:
                 nvfp4_compressed_tensors = self._is_nvfp4_compressed_tensors(quant_method, quant_format, groups)
 
                 if nvfp4_compressed_tensors:
-                    fp8_configs = [
-                        g["weights"] for g in groups.values()
-                        if isinstance(g, dict) and g.get("format") == "float-quantized"
-                        and g.get("weights", {}).get("type") == "float"
-                        and g.get("weights", {}).get("num_bits") == 8
-                    ]
-                    dequant_fp8(fp8_configs)
+                    dequant_fp8()
                 elif len(groups) > 1:
                     raise NotImplementedError("Can't handle multiple config groups for compressed-tensors yet")
                 elif quant_format == "float-quantized" or quant_format == "int-quantized" or quant_format == "naive-quantized":
                     weight_config = tuple(groups.values())[0]["weights"]
                     block_size = weight_config.get("block_structure", None)
                     strategy = weight_config.get("strategy")
+                    assert strategy == "channel" or strategy == "block"
+                    assert weight_config.get("group_size") is None  # didn't find a model using this yet
                     is_fp8 = (
                         quant_format == "float-quantized"
                         and weight_config.get("type") == "float"
                         and weight_config.get("num_bits") == 8
                     )
-                    assert strategy in ("channel", "block") or (is_fp8 and strategy == "tensor")
-                    assert weight_config.get("group_size") is None  # didn't find a model using this yet
                     for name in self.model_tensors.keys():
                         if name.endswith(".weight_scale"):
                             weight_name = name.removesuffix("_scale")
                             w = self.model_tensors[weight_name]
                             s = self.model_tensors[name]
-                            self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+                            self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), block_size)
                             tensors_to_remove.append(name)
                             if self._fp8_as_q8 and is_fp8:
                                 self._fp8_dequantized.add(weight_name)
-                        if is_fp8 and name.endswith((".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")):
-                            tensors_to_remove.append(name)
                 elif quant_format == "pack-quantized":
                     weight_config = tuple(groups.values())[0]["weights"]
                     assert weight_config.get("strategy") == "group"
@@ -616,148 +581,6 @@ class ModelBase:
         for name, value in new_tensors.items():
             self.model_tensors[name] = value
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        new_name = self.tensor_map.get_name(name, try_suffixes=(".weight",))
-        return (new_name,) if new_name is not None else ()
-
-    def _prepare_fp8_e4m3_tensors(self):
-        if not self._preserve_fp8:
-            return
-
-        scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
-        input_scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
-        consumed: list[str] = []
-        candidates: dict[str, tuple[str, Tensor, tuple[str, ...], torch.Size]] = {}
-        weight_dtypes: dict[str, torch.dtype] = {}
-        scaled_weights: set[str] = set()
-        expert_pattern = re.compile(r"(\.[^.]*experts)\.(\d+)\.")
-        scale_types = {
-            gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V,
-            gguf.MODEL_TENSOR.ATTN_OUT, gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE,
-            gguf.MODEL_TENSOR.FFN_GATE, gguf.MODEL_TENSOR.FFN_UP, gguf.MODEL_TENSOR.FFN_DOWN,
-            gguf.MODEL_TENSOR.FFN_GATE_SHEXP, gguf.MODEL_TENSOR.FFN_UP_SHEXP, gguf.MODEL_TENSOR.FFN_DOWN_SHEXP,
-            gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
-            gguf.MODEL_TENSOR.SSM_IN, gguf.MODEL_TENSOR.SSM_OUT, gguf.MODEL_TENSOR.SSM_ALPHA, gguf.MODEL_TENSOR.SSM_BETA,
-            gguf.MODEL_TENSOR.NEXTN_EH_PROJ, gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, gguf.MODEL_TENSOR.OUTPUT,
-        }
-
-        for scale_name in list(self.model_tensors.keys()):
-            weight_name = None
-            if scale_name.endswith(".weight_scale"):
-                weight_name = scale_name.removesuffix("_scale")
-            elif scale_name.endswith("_scale_inv"):
-                weight_name = scale_name.removesuffix("_scale_inv")
-            elif scale_name.endswith(".qscale_weight"):
-                weight_name = scale_name.removesuffix("qscale_weight") + "weight"
-
-            if weight_name is None or weight_name not in self.model_tensors:
-                continue
-
-            scaled_weights.add(weight_name)
-            weight = self.model_tensors[weight_name]()
-            weight_dtypes[weight_name] = weight.dtype
-            if weight.dtype != torch.float8_e4m3fn or weight.ndim < 2:
-                continue
-
-            scale = self.model_tensors[scale_name]()
-            if prod(scale.shape) != 1:
-                continue
-
-            merged_name = expert_pattern.sub(r"\1.", weight_name)
-            new_names = self._map_fp8_weight_names(merged_name)
-            if not new_names:
-                continue
-            if any((mapped := self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",))) is None or mapped[0] not in scale_types for new_name in new_names):
-                continue
-            bid = next((int(part) for part in new_names[0].split(".") if part.isdecimal()), None)
-            self._fp8_e4m3_preserved.add(weight_name)
-            if any(new_name.endswith("_norm.weight") or self.tensor_force_quant(weight_name, new_name, bid, weight.ndim) != gguf.GGMLQuantizationType.F8_E4M3 for new_name in new_names):
-                self._fp8_e4m3_preserved.discard(weight_name)
-                continue
-            candidates[weight_name] = (scale_name, scale, new_names, weight.shape)
-
-        expert_groups: dict[str, list[str]] = {}
-        for name in self.model_tensors:
-            if (match := expert_pattern.search(name)) and name.endswith(".weight"):
-                expert_groups.setdefault(name[:match.end(1)], []).append(name)
-        for names in expert_groups.values():
-            for name in names:
-                if name not in weight_dtypes:
-                    weight_dtypes[name] = self.model_tensors[name]().dtype
-            dtypes = {weight_dtypes[name] for name in names}
-            if torch.float8_e4m3fn not in dtypes:
-                continue
-            if dtypes == {torch.float8_e4m3fn} and not any(name in scaled_weights for name in names):
-                continue
-            # The converter merges expert projections together.
-            if not all(name in candidates for name in names):
-                for name in names:
-                    candidates.pop(name, None)
-                    self._fp8_e4m3_preserved.discard(name)
-                    if name not in scaled_weights:
-                        gen = self.model_tensors[name]
-                        self.model_tensors[name] = lambda gen=gen: gen().float()
-
-        for weight_name, (scale_name, scale, new_names, shape) in candidates.items():
-            scale = LazyTorchTensor.to_eager(scale).float().flatten()
-
-            weight_prefix = weight_name.removesuffix(".weight")
-            # Transformers fine-grained FP8 uses activation_scale while ModelOpt uses input_scale.
-            # Ref: https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/mistral3.py#L357-L361
-            input_scale_name = next((name for name in (
-                weight_prefix + ".input_scale",
-                weight_prefix + ".activation_scale",
-                weight_prefix + "_activation_scale",
-                weight_prefix + ".qscale_act",
-            ) if name in self.model_tensors), None)
-            input_scale = None
-            if input_scale_name is not None:
-                input_scale = self.model_tensors[input_scale_name]()
-                if prod(input_scale.shape) != 1:
-                    raise ValueError(f"FP8 input scale {input_scale_name!r} must be a scalar")
-                input_scale = LazyTorchTensor.to_eager(input_scale).float().flatten()
-                consumed.append(input_scale_name)
-
-            consumed.append(scale_name)
-
-            expert_match = expert_pattern.search(weight_name)
-            if expert_match:
-                expert_id = int(expert_match.group(2))
-                for new_name in new_names:
-                    target_name = new_name.replace(".weight", ".scale")
-                    entries = scale_tensors.setdefault(target_name, [])
-                    assert isinstance(entries, list)
-                    cast(list[tuple[int, float]], entries).append((expert_id, float(scale[0])))
-                    if input_scale is not None:
-                        target_name = new_name.replace(".weight", ".input_scale")
-                        entries = input_scale_tensors.setdefault(target_name, [])
-                        assert isinstance(entries, list)
-                        cast(list[tuple[int, float]], entries).append((expert_id, float(input_scale[0])))
-            else:
-                for new_name in new_names:
-                    mapped = self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",)) if len(shape) == 3 else None
-                    is_packed_expert = mapped is not None and mapped[0] in (
-                        gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
-                    )
-                    # Packed expert weights need one scale value per expert.
-                    n_scales = shape[0] if is_packed_expert else 1
-                    scale_tensors[new_name.replace(".weight", ".scale")] = np.repeat(scale.numpy(), n_scales)
-                    if input_scale is not None:
-                        input_scale_tensors[new_name.replace(".weight", ".input_scale")] = np.repeat(input_scale.numpy(), n_scales)
-
-        for name in consumed:
-            self.model_tensors.pop(name, None)
-
-        for name, values in chain(scale_tensors.items(), input_scale_tensors.items()):
-            if isinstance(values, list):
-                values.sort(key=lambda item: item[0])
-                scale = np.array([item[1] for item in values], dtype=np.float32)
-            else:
-                scale = values.astype(np.float32)
-            if not np.allclose(scale, 1.0, atol=1e-6):
-                logger.info(f"  + {name} (FP8 scale, shape [{scale.size}])")
-                self.gguf_writer.add_tensor(name, scale)
-
     @classmethod
     def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
         name, gen = item
@@ -802,26 +625,6 @@ class ModelBase:
             raise ValueError(f"Can not map tensor {name!r}")
         return new_name
 
-    def prepare_gate_up_fusion(self) -> None:
-        self._unfusable_gate_up_layers.clear()
-        if not self.fuse_gate_up_exps or self._fp8_as_q8:
-            return
-
-        for name, gen in self.model_tensors.items():
-            merged_name = re.sub(r"(\.experts)\.\d+\.", r"\1.", name)
-            mapped = self.tensor_map.get_type_and_name(merged_name, try_suffixes=(".weight",))
-            if mapped is None:
-                continue
-            tensor_type, new_name = mapped
-            if tensor_type not in (gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_GATE_UP_EXP) or not new_name.endswith(".weight"):
-                continue
-            if gen().dtype == torch.float8_e4m3fn:
-                bid = next(int(part) for part in new_name.split(".") if part.isdecimal())
-                self._unfusable_gate_up_layers.add(bid)
-
-        for bid in sorted(self._unfusable_gate_up_layers):
-            logger.warning(f"Skipping --fuse-gate-up-exps for layer {bid}: fusion of preserved FP8 expert projections is not supported")
-
     def prepare_qkv_fusion(self) -> None:
         self._fusable_qkv_weight_layers.clear()
         self._fusable_qkv_bias_layers.clear()
@@ -835,7 +638,6 @@ class ModelBase:
         }
         weights: dict[int, set[gguf.MODEL_TENSOR]] = {}
         biases: dict[int, set[gguf.MODEL_TENSOR]] = {}
-        fp8_layers: set[int] = set()
 
         for name in self.model_tensors:
             mapped = self.tensor_map.get_type_and_name(name, try_suffixes=(".weight", ".bias"))
@@ -850,19 +652,12 @@ class ModelBase:
                 continue
             if new_name.endswith(".weight"):
                 weights.setdefault(bid, set()).add(tensor_type)
-                if not self._fp8_as_q8 and self.model_tensors[name]().dtype == torch.float8_e4m3fn:
-                    fp8_layers.add(bid)
             elif new_name.endswith(".bias"):
                 biases.setdefault(bid, set()).add(tensor_type)
 
         for bid, weight_types in weights.items():
             bias_types = biases.get(bid, set())
             if weight_types == qkv_types and (not bias_types or bias_types == qkv_types):
-                # NVFP4 weights use separate reordering and repacking hooks and are exported before this fusion pass.
-                # Separate FP8 scales cannot be represented by one fused QKV scale.
-                if bid in fp8_layers:
-                    logger.warning(f"Skipping --fuse-qkv for layer {bid}: fusion of preserved FP8 projections is not supported")
-                    continue
                 self._fusable_qkv_weight_layers.add(bid)
                 if bias_types:
                     self._fusable_qkv_bias_layers.add(bid)
@@ -1051,8 +846,7 @@ class ModelBase:
         new_name = self.map_tensor_name(name)
 
         # Handle gate/up expert tensor fusion if enabled
-        # Preserved FP8 gate/up scales must stay with their separate projections.
-        if self.fuse_gate_up_exps and bid is not None and bid not in self._unfusable_gate_up_layers:
+        if self.fuse_gate_up_exps and bid is not None:
             if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_GATE_EXP, bid):
                 self._gate_exp_buffer[bid] = data_torch
             elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_UP_EXP, bid):
@@ -1109,46 +903,8 @@ class ModelBase:
 
         return [(new_name, data_torch)]
 
-    def _tensor_requires_f32(self, new_name: str, bid: int | None, n_dims: int) -> bool:
-        # Conditions should closely match those in llama_model_quantize_internal in llama.cpp.
-        return n_dims <= 1 or new_name.endswith("_norm.weight") or (
-            any(
-                self.match_model_tensor_name(new_name, key, bid)
-                for key in (
-                    gguf.MODEL_TENSOR.FFN_GATE_INP,
-                    gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
-                    gguf.MODEL_TENSOR.POS_EMBD,
-                    gguf.MODEL_TENSOR.TOKEN_TYPES,
-                    gguf.MODEL_TENSOR.SSM_CONV1D,
-                    gguf.MODEL_TENSOR.SHORTCONV_CONV,
-                    gguf.MODEL_TENSOR.TIME_MIX_FIRST,
-                    gguf.MODEL_TENSOR.TIME_MIX_W1,
-                    gguf.MODEL_TENSOR.TIME_MIX_W2,
-                    gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,
-                    gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,
-                    gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,
-                    gguf.MODEL_TENSOR.POSNET_NORM1,
-                    gguf.MODEL_TENSOR.POSNET_NORM2,
-                    gguf.MODEL_TENSOR.V_ENC_EMBD_POS,
-                    gguf.MODEL_TENSOR.A_ENC_EMBD_POS,
-                    gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,
-                    gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,
-                    # Kimi KDA conv weights should be F32
-                    gguf.MODEL_TENSOR.SSM_CONV1D_Q,
-                    gguf.MODEL_TENSOR.SSM_CONV1D_K,
-                    gguf.MODEL_TENSOR.SSM_CONV1D_V,
-                    # DSA indexer weights should be F32
-                    gguf.MODEL_TENSOR.INDEXER_PROJ,
-                )
-            )
-            or new_name[-7:] not in (".weight", ".lora_a", ".lora_b")
-        )
-
     def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        if name in self._fp8_e4m3_preserved and n_dims >= 2:
-            if self._tensor_requires_f32(new_name, bid, n_dims):
-                return gguf.GGMLQuantizationType.F32
-            return gguf.GGMLQuantizationType.F8_E4M3
+        del new_name, bid  # unused
         # Force FP8-original tensors to Q8_0 when requested; Q8_0 is faster than F16/BF16.
         if self._fp8_as_q8 and name in self._fp8_dequantized and n_dims >= 2:
             return gguf.GGMLQuantizationType.Q8_0
@@ -1279,8 +1035,7 @@ class ModelBase:
         expert_scales: dict[tuple[int, str], list[tuple[int, float]]] = {}
         expert_input_scales: dict[tuple[int, str], list[tuple[int, float]]] = {}
         expert_shapes: dict[tuple[int, str], list[int]] = {}
-        expert_names: dict[tuple[int, str], str] = {}
-        n_experts = self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True) or 0
+        n_experts = self.find_hparam(["num_local_experts", "num_experts"], optional=True) or 0
         consumed: list[str] = []
 
         for name in self.model_tensors.keys():
@@ -1291,17 +1046,14 @@ class ModelBase:
             input_scale_name = name.replace(".weight", ".input_scale")
             if scale_name not in self.model_tensors:
                 continue
-            weight = self.model_tensors[name]()
-            scale = self.model_tensors[scale_name]()
+            # Force eager materialization of lazy tensors
+            weight = LazyTorchTensor.to_eager(self.model_tensors[name]())
+            scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]())
 
-            # Leave FP8 weights and their scales for _prepare_fp8_e4m3_tensors.
-            if weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2) or scale.ndim < 2:
+            # Skip non-NVFP4 tensors(e.g. 1D scale, or float8 weight)
+            if scale.ndim < 2 or weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
                 continue
-            if weight.ndim != 2 or scale.shape[0] != weight.shape[0] or scale.shape[1] * 8 != weight.shape[1]:
-                raise ValueError(f"NVFP4 weight {name!r} has incompatible shapes: weight {list(weight.shape)}, scale {list(scale.shape)}")
 
-            weight = LazyTorchTensor.to_eager(weight)
-            scale = LazyTorchTensor.to_eager(scale)
             scale2 = LazyTorchTensor.to_eager(self.model_tensors.get(scale2_name, lambda: torch.tensor(1.0))())
             input_scale = LazyTorchTensor.to_eager(self.model_tensors.get(input_scale_name, lambda: torch.tensor(1.0))())
 
@@ -1328,7 +1080,6 @@ class ModelBase:
                     expert_scales[key] = []
                     expert_input_scales[key] = []
                     expert_shapes[key] = shape
-                    expert_names[key] = name
                 expert_blocks[key].append((expert_id, raw.copy()))
                 # Collect per-expert scale2 (scalar per expert)
                 expert_scales[key].append((expert_id, float(scale2.float().sum())))
@@ -1337,13 +1088,13 @@ class ModelBase:
 
                 # Flush when all experts for this (layer, proj) are collected
                 if n_experts > 0 and len(expert_blocks[key]) >= n_experts:
-                    self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type)
+                    self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type)
             else:
                 self._repack_nvfp4(name, weight, scale, scale2, input_scale)
 
         # Flush any remaining experts (fallback if n_experts was unknown)
         for bid, proj_type in list(expert_blocks.keys()):
-            self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type)
+            self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type)
 
         # Remove consumed tensors so get_tensors/modify_tensors won't see them
         for name in consumed:
@@ -1354,7 +1105,7 @@ class ModelBase:
             if name.endswith((".k_scale", ".v_scale")):
                 del self.model_tensors[name]
 
-    def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type):
+    def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type):
         experts = expert_blocks.pop(key)
         scales = expert_scales.pop(key)
         input_scales = expert_input_scales.pop(key)
@@ -1366,7 +1117,7 @@ class ModelBase:
         new_name = self.map_tensor_name(merged_name)
         logger.info(f"Repacked {new_name} with shape [{len(experts)}, {shape[0]}, {shape[1]}] and quantization NVFP4")
         self.gguf_writer.add_tensor(new_name, merged, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
-        self._tag_prec_a4(expert_names.pop(key), new_name)
+        self._tag_prec_a4(merged_name, new_name)
 
         scales.sort(key=lambda x: x[0])
         self._write_scales_tensor(new_name.replace(".weight", ".scale"), [s[1] for s in scales])
@@ -1458,10 +1209,8 @@ class ModelBase:
                             self.model_tensors[input_scale_name] = inverse_scale(self.model_tensors.pop(name))
             self._generate_nvfp4_tensors()
 
-        self._prepare_fp8_e4m3_tensors()
         self.dequant_model()
 
-        self.prepare_gate_up_fusion()
         self.prepare_qkv_fusion()
 
         # Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
@@ -1475,21 +1224,10 @@ class ModelBase:
             if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
                 continue
 
-            # FP8 transforms must only reorder values; norms and vectors use numeric F32 transforms.
-            preserve_fp8_e4m3 = self._preserve_fp8 and data_torch.dtype == torch.float8_e4m3fn and data_torch.ndim >= 2 and not name.endswith(("norm.weight", "norm_f.weight"))
-            if preserve_fp8_e4m3:
-                for new_name in self._map_fp8_weight_names(name):
-                    mapped_bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None)
-                    if self._tensor_requires_f32(new_name, mapped_bid, data_torch.ndim):
-                        preserve_fp8_e4m3 = False
-                        break
-                if preserve_fp8_e4m3:
-                    self._fp8_e4m3_preserved.add(name)
-
             old_dtype = data_torch.dtype
 
             # convert any unsupported data types to float32
-            if data_torch.dtype not in (torch.float16, torch.float32) and name not in self._fp8_e4m3_preserved:
+            if data_torch.dtype not in (torch.float16, torch.float32):
                 data_torch = data_torch.to(torch.float32)
 
             # use the first number-like part of the tensor name as the block id
@@ -1500,14 +1238,51 @@ class ModelBase:
                     break
 
             for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):
-                n_dims = len(data_torch.shape)
+                # TODO: why do we squeeze here?
+                # data = data_torch.squeeze().numpy()
+                data = data_torch.numpy()
+
+                n_dims = len(data.shape)
                 data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)
 
                 # Most of the codebase that takes in 1D tensors or norms only handles F32 tensors
                 if n_dims <= 1 or new_name.endswith("_norm.weight"):
                     data_qtype = gguf.GGMLQuantizationType.F32
 
-                if data_qtype is False and self._tensor_requires_f32(new_name, bid, n_dims):
+                # Conditions should closely match those in llama_model_quantize_internal in llama.cpp
+                # Some tensor types are always in float32
+                if data_qtype is False and (
+                    any(
+                        self.match_model_tensor_name(new_name, key, bid)
+                        for key in (
+                            gguf.MODEL_TENSOR.FFN_GATE_INP,
+                            gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
+                            gguf.MODEL_TENSOR.POS_EMBD,
+                            gguf.MODEL_TENSOR.TOKEN_TYPES,
+                            gguf.MODEL_TENSOR.SSM_CONV1D,
+                            gguf.MODEL_TENSOR.SHORTCONV_CONV,
+                            gguf.MODEL_TENSOR.TIME_MIX_FIRST,
+                            gguf.MODEL_TENSOR.TIME_MIX_W1,
+                            gguf.MODEL_TENSOR.TIME_MIX_W2,
+                            gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,
+                            gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,
+                            gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,
+                            gguf.MODEL_TENSOR.POSNET_NORM1,
+                            gguf.MODEL_TENSOR.POSNET_NORM2,
+                            gguf.MODEL_TENSOR.V_ENC_EMBD_POS,
+                            gguf.MODEL_TENSOR.A_ENC_EMBD_POS,
+                            gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,
+                            gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,
+                            # Kimi KDA conv weights should be F32
+                            gguf.MODEL_TENSOR.SSM_CONV1D_Q,
+                            gguf.MODEL_TENSOR.SSM_CONV1D_K,
+                            gguf.MODEL_TENSOR.SSM_CONV1D_V,
+                            # DSA indexer weights should be F32
+                            gguf.MODEL_TENSOR.INDEXER_PROJ,
+                        )
+                    )
+                    or new_name[-7:] not in (".weight", ".lora_a", ".lora_b")
+                ):
                     data_qtype = gguf.GGMLQuantizationType.F32
 
                 if data_qtype is False and any(
@@ -1545,22 +1320,16 @@ class ModelBase:
                     else:
                         raise ValueError(f"Unknown file type: {self.ftype.name}")
 
-                if data_torch.dtype == torch.float8_e4m3fn:
-                    data = data_torch.view(torch.uint8).numpy() if data_qtype == gguf.GGMLQuantizationType.F8_E4M3 else data_torch.float().numpy()
-                else:
-                    data = data_torch.numpy()
-
                 # a chunked tensor quantizes as one chunk at a time, while it is written
                 quantize = data.quantize if isinstance(data, gguf.LazyChunkedTensor) else (
                     lambda qtype, d=data: gguf.quants.quantize(d, qtype))
 
-                if not (data_qtype == gguf.GGMLQuantizationType.F8_E4M3 and data.dtype == np.uint8):
-                    try:
-                        data = quantize(data_qtype)
-                    except gguf.QuantError as e:
-                        logger.warning("%s, %s", e, "falling back to F16")
-                        data_qtype = gguf.GGMLQuantizationType.F16
-                        data = quantize(data_qtype)
+                try:
+                    data = quantize(data_qtype)
+                except gguf.QuantError as e:
+                    logger.warning("%s, %s", e, "falling back to F16")
+                    data_qtype = gguf.GGMLQuantizationType.F16
+                    data = quantize(data_qtype)
 
                 shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
 
@@ -1601,8 +1370,6 @@ class ModelBase:
                 self.ftype = gguf.LlamaFileType.MOSTLY_NVFP4
             elif self._is_mxfp4:
                 self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
-            elif self._fp8_e4m3_preserved:
-                self.ftype = gguf.LlamaFileType.MOSTLY_F8_E4M3
 
         # Generate parameter weight class (useful for leader boards) if not yet determined
         if self.metadata.size_label is None and total_params > 0:
@@ -2118,9 +1885,6 @@ class TextModel(ModelBase):
         if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4":
             # ref: https://huggingface.co/evilfreelancer/ruGPT3XL
             res = "gpt-2"
-        if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9":
-            # ref: https://huggingface.co/Lernex/Metis-1.6-Logos
-            res = "gpt-2"
         if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
             # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
             res = "lfm2"
@@ -2908,13 +2672,7 @@ class TextModel(ModelBase):
         )
 
         if vocab.tokenizer_type == MistralTokenizerType.tekken:
-            with open(vocab.fname_tokenizer, encoding="utf-8") as f:
-                pattern = json.load(f)["config"]["pattern"]
-            # newer tekken.json split numbers into right-aligned groups of 3 digits instead of single digits
-            if "\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))" in pattern:
-                self.gguf_writer.add_tokenizer_pre("tekken-num3")
-            else:
-                self.gguf_writer.add_tokenizer_pre("tekken")
+            self.gguf_writer.add_tokenizer_pre("tekken")
             self.gguf_writer.add_token_merges(
                 vocab.extract_vocab_merges_from_model()
             )
@@ -3391,7 +3149,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
     # Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
     # Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
     # latent MoE, situ, ...). Route on the top-level architecture instead.
-    if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
+    if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"):
         return arch
 
     # if "architectures" is found in the sub-config, use that instead

conversion/chatterbox.py

@@ -1,471 +0,0 @@
-# Chatterbox (ResembleAI) turbo and multilingual v3, from the raw checkpoints of the official repos
-# the checkpoints ship no config.json, the variant is detected from the talker file and the hparams come from the reference code
-# talker: text_emb and speech_emb are fused into one [text | speech] vocab, speech token i is <|speech_i|>
-# mmproj: the s3gen sidecar, the voice and conditioning encoders, the position and speech tables, and the conds.pt voice
-
-from __future__ import annotations
-
-import json
-import math
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Iterable
-
-import torch
-import torch.nn.functional as F
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger
-from .llama import LlamaModel
-
-# checkpoint file prefixes, the files are "<prefix>.safetensors"
-TURBO_TALKER = "t3_turbo_v1"
-MTL_TALKER   = "t3_mtl23ls_v3"
-TURBO_S3GEN  = "s3gen_meanflow"
-MTL_S3GEN    = "s3gen_v3"
-VE           = "ve"
-
-# relative speech ids shared by both variants (start/stop_speech_token in the reference)
-SPEECH_BOS = 6561
-SPEECH_EOS = 6562
-
-# sampling defaults of the reference, samplers it does not use are disabled so the common defaults do not apply
-TURBO_TOP_K = 1000
-MTL_TOP_K   = 0
-TURBO_SAMPLING = {"min_p": 0.0,  "top_p": 0.95, "temp": 0.8, "penalty_repeat": 1.2}
-MTL_SAMPLING   = {"min_p": 0.05, "top_p": 1.0,  "temp": 0.8, "penalty_repeat": 1.2}
-# the reference repetition penalty covers the whole generated history
-PENALTY_LAST_N = -1
-
-
-def _has_file(dir_model: Path, prefix: str) -> bool:
-    return (dir_model / f"{prefix}.safetensors").is_file()
-
-
-def _is_turbo(dir_model: Path) -> bool:
-    if _has_file(dir_model, TURBO_TALKER):
-        return True
-    if _has_file(dir_model, MTL_TALKER):
-        return False
-    raise FileNotFoundError(f"no chatterbox talker checkpoint in {dir_model}")
-
-
-# talker hparams of the reference (GPT2_MEDIUM_CONFIG / LLAMA_520M_CONFIG_DICT, T3Config)
-TURBO_TEXT_CONFIG = {
-    "n_layer": 24, "n_embd": 1024, "n_inner": 4096, "n_head": 16, "n_ctx": 8196, "layer_norm_epsilon": 1e-05,
-    "vocab_size": 50276, "speech_vocab_size": 6563,
-}
-MTL_TEXT_CONFIG = {
-    "num_hidden_layers": 30, "hidden_size": 1024, "intermediate_size": 4096, "num_attention_heads": 16,
-    "num_key_value_heads": 16, "head_dim": 64, "max_position_embeddings": 131072, "rms_norm_eps": 1e-05,
-    "rope_theta": 500000.0,
-    "rope_scaling": {"factor": 8.0, "high_freq_factor": 4.0, "low_freq_factor": 1.0, "original_max_position_embeddings": 8192},
-    "vocab_size": 2454, "speech_vocab_size": 8194,
-}
-
-
-def _is_chatterbox_checkpoint(dir_model: Path) -> bool:
-    return _has_file(dir_model, VE) and (_has_file(dir_model, TURBO_TALKER) or _has_file(dir_model, MTL_TALKER))
-
-
-@ModelBase.register_hparams_loader(_is_chatterbox_checkpoint)
-def _load_chatterbox_hparams(dir_model: Path) -> dict[str, Any]:
-    turbo = _is_turbo(dir_model)
-    logger.info(f"gguf: detected Chatterbox {'turbo' if turbo else 'multilingual v3'} checkpoint")
-    text_config = dict(TURBO_TEXT_CONFIG if turbo else MTL_TEXT_CONFIG)
-    # the mmproj tensors are named explicitly, they do not go through the block tensor map
-    return {"architectures": ["ChatterboxModel"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}}
-
-
-@ModelBase.register("ChatterboxModel")
-class ChatterboxTalkerModel(TextModel):
-    model_arch = gguf.MODEL_ARCH.LLAMA  # multilingual; the turbo constructor switches to GPT2
-
-    def __init__(self, dir_model: Path, *args, **kwargs):
-        self.is_turbo = _is_turbo(dir_model)
-        if self.is_turbo:
-            self.model_arch = gguf.MODEL_ARCH.GPT2
-        super().__init__(dir_model, *args, **kwargs)
-        self._text_embd: Tensor | None = None
-        self._speech_embd: Tensor | None = None
-        self._text_head: Tensor | None = None
-        self._speech_head: Tensor | None = None
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        return self.index_local_tensors(TURBO_TALKER if self.is_turbo else MTL_TALKER)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        # the reference feeds inputs_embeds, the transformer input embeddings are unused
-        if name in ("tfmr.wte.weight", "tfmr.embed_tokens.weight"):
-            return None
-        if name.startswith(("cond_enc.", "text_pos_emb.", "speech_pos_emb.")):
-            return None
-        return super().filter_tensors(item)
-
-    def set_vocab(self):
-        if self.is_turbo:
-            self._set_vocab_turbo()
-        else:
-            self._set_vocab_mtl()
-
-    def _speech_token_names(self, n_speech: int) -> list[str]:
-        return [f"<|speech_{i}|>" for i in range(n_speech)]
-
-    def _set_vocab_turbo(self):
-        # stock GPT-2 BPE from the checkpoint dir, extended with the speech tokens
-        tokens, toktypes, tokpre = self.get_vocab_base()
-        n_text = len(tokens)
-        n_speech = self.hparams["speech_vocab_size"]
-        speech = self._speech_token_names(n_speech)
-        tokens += speech
-        toktypes += [gguf.TokenType.CONTROL] * n_speech
-
-        with open(self.dir_model / "merges.txt", "r", encoding="utf-8") as f:
-            merges = [line.rstrip("\n") for line in f if line.strip() and not line.startswith("#version")]
-
-        self._write_vocab(tokens, toktypes, merges, tokpre, n_text)
-
-    def _write_vocab(self, tokens: list[str], toktypes: list[int], merges: list[str], tokpre: str, n_text: int):
-        self.gguf_writer.add_tokenizer_model("gpt2")
-        self.gguf_writer.add_tokenizer_pre(tokpre)
-        self.gguf_writer.add_token_list(tokens)
-        self.gguf_writer.add_token_types(toktypes)
-        self.gguf_writer.add_token_merges(merges)
-
-        self.gguf_writer.add_bos_token_id(n_text + SPEECH_BOS)
-        self.gguf_writer.add_eos_token_id(n_text + SPEECH_EOS)
-        self.gguf_writer.add_add_bos_token(False)
-        self.gguf_writer.add_add_eos_token(False)
-
-        # the reference samples the speech head only
-        self.gguf_writer.add_suppress_tokens(list(range(n_text)))
-
-    def _set_vocab_mtl(self):
-        # the reference BPE is char-level, it is re-encoded to byte-level with extra merges that rebuild each multi-byte char
-        # its lowercasing is folded into the embedding table, so the prompt needs no text preprocessing
-        with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f:
-            tok = json.load(f)
-
-        byte_map = gguf.vocab.bytes_to_unicode()
-
-        def enc(s: str) -> str:
-            return "".join(byte_map[b] for b in s.encode("utf-8"))
-
-        n_text = self.hparams["vocab_size"]
-        tokens: list[str] = [f"[unused_{i}]" for i in range(n_text)]
-        toktypes = [int(gguf.TokenType.UNUSED)] * n_text
-        char_merges: list[str] = []
-        for t, i in tok["model"]["vocab"].items():
-            if t == " ":
-                # the reference substitutes [SPACE] before encoding, the raw space token is never used
-                continue
-            tokens[i] = enc(t)
-            toktypes[i] = int(gguf.TokenType.NORMAL)
-            if len(t) == 1 and len(t.encode("utf-8")) > 1:
-                parts = [byte_map[b] for b in t.encode("utf-8")]
-                for k in range(1, len(parts)):
-                    char_merges.append("".join(parts[:k]) + " " + parts[k])
-        for entry in tok.get("added_tokens", []):
-            tokens[entry["id"]] = entry["content"]
-            toktypes[entry["id"]] = int(gguf.TokenType.CONTROL)
-
-        # raw spaces tokenize to [SPACE] directly
-        space_id = tok["model"]["vocab"]["[SPACE]"]
-        tokens[space_id] = enc(" ")
-        toktypes[space_id] = int(gguf.TokenType.NORMAL)
-
-        n_speech = self.hparams["speech_vocab_size"]
-        tokens += self._speech_token_names(n_speech)
-        toktypes += [int(gguf.TokenType.CONTROL)] * n_speech
-
-        # chars are atomic in the reference, they must form before any of its merges
-        merges = char_merges
-        for m in tok["model"].get("merges", []):
-            a, b = m if isinstance(m, list) else m.split(" ")
-            merges.append(enc(a) + " " + enc(b))
-
-        self._write_vocab(tokens, toktypes, merges, "default", n_text)
-
-    def _mtl_case_fold_pairs(self) -> list[tuple[int, int]]:
-        # (upper_id, lower_id) of the single-char vocab entries, for every script in the vocab
-        with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f:
-            vocab = json.load(f)["model"]["vocab"]
-        pairs: list[tuple[int, int]] = []
-        for t, i in vocab.items():
-            if len(t) != 1:
-                continue
-            low = t.lower()
-            if low != t and len(low) == 1 and low in vocab:
-                pairs.append((i, vocab[low]))
-        return pairs
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        sampling = TURBO_SAMPLING if self.is_turbo else MTL_SAMPLING
-        self.gguf_writer.add_sampling_top_k(TURBO_TOP_K if self.is_turbo else MTL_TOP_K)
-        self.gguf_writer.add_sampling_min_p(sampling["min_p"])
-        self.gguf_writer.add_sampling_top_p(sampling["top_p"])
-        self.gguf_writer.add_sampling_temp(sampling["temp"])
-        self.gguf_writer.add_sampling_penalty_repeat(sampling["penalty_repeat"])
-        self.gguf_writer.add_sampling_penalty_last_n(PENALTY_LAST_N)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if self.is_turbo:
-            return
-        # llama3 rope scaling baked into the rope_freqs factors tensor
-        rp = self.hparams["rope_scaling"]
-        dim = self.hparams["head_dim"]
-        base = self.hparams["rope_theta"]
-        freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
-        factor = rp["factor"]
-        low_freq_wavelen = rp["original_max_position_embeddings"] / rp["low_freq_factor"]
-        high_freq_wavelen = rp["original_max_position_embeddings"] / rp["high_freq_factor"]
-        rope_factors = []
-        for freq in freqs:
-            wavelen = 2 * math.pi / freq
-            if wavelen < high_freq_wavelen:
-                rope_factors.append(1)
-            elif wavelen > low_freq_wavelen:
-                rope_factors.append(factor)
-            else:
-                smooth = (rp["original_max_position_embeddings"] / wavelen - rp["low_freq_factor"]) / (rp["high_freq_factor"] - rp["low_freq_factor"])
-                rope_factors.append(1 / ((1 - smooth) / factor + smooth))
-        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
-
-    def _maybe_emit_fused(self) -> Iterable[tuple[str, Tensor]]:
-        if self._text_embd is not None and self._speech_embd is not None:
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD),
-                   torch.cat([self._text_embd, self._speech_embd], dim=0))
-            self._text_embd = None
-            self._speech_embd = None
-        if self._text_head is not None and self._speech_head is not None:
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT),
-                   torch.cat([self._text_head, self._speech_head], dim=0))
-            self._text_head = None
-            self._speech_head = None
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # generate_extra_tensors output comes back through here with its final name
-        if name.startswith("rope_freqs"):
-            yield (name, data_torch)
-            return
-        # fused [text | speech] vocab: embeddings and output head
-        if name == "text_emb.weight":
-            if not self.is_turbo:
-                # the reference lowercases its input, the uppercase rows take the lowercase ones
-                fold = list(range(data_torch.shape[0]))
-                for upper, lower in self._mtl_case_fold_pairs():
-                    fold[upper] = lower
-                data_torch = data_torch[torch.tensor(fold)]
-            self._text_embd = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "speech_emb.weight":
-            self._speech_embd = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "text_head.weight":
-            self._text_head = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "speech_head.weight":
-            self._speech_head = data_torch
-            yield from self._maybe_emit_fused()
-            return
-        if name == "speech_head.bias":
-            # the text head has no bias, its zone is suppressed at sampling anyway
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT, suffix=".bias"), F.pad(data_torch, (self.hparams["vocab_size"], 0)))
-            return
-
-        assert name.startswith("tfmr.")
-        name = name[len("tfmr."):]
-
-        if self.is_turbo:
-            # HF GPT-2 layout: Conv1D style weights are stored transposed
-            if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight")):
-                data_torch = data_torch.transpose(1, 0)
-            yield (self.map_tensor_name(name), data_torch)
-            return
-
-        if name.endswith("q_proj.weight"):
-            data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], None)
-        if name.endswith("k_proj.weight"):
-            data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], self.hparams["num_key_value_heads"])
-        yield (self.map_tensor_name("model." + name), data_torch)
-
-
-@ModelBase.register("ChatterboxModel")
-class ChatterboxMmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    def __init__(self, dir_model: Path, *args, **kwargs):
-        self.is_turbo = _is_turbo(dir_model)
-        super().__init__(dir_model, *args, **kwargs)
-        self._wnorm_g: dict[str, Tensor] = {}
-        self._wnorm_v: dict[str, Tensor] = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        return self.global_config.get("audio_config")
-
-    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
-        tensors: dict[str, Callable[[], Tensor]] = {}
-        for prefix in (TURBO_S3GEN if self.is_turbo else MTL_S3GEN, VE, TURBO_TALKER if self.is_turbo else MTL_TALKER):
-            tensors.update(self.index_local_tensors(prefix))
-        return tensors
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        # the s3gen, ve and talker checkpoints have disjoint top-level names
-        name, gen = item
-        new_name: str | None = None
-        # batchnorm bookkeeping (over the gguf name length cap) and the tokenizer dsp buffers are unused
-        if name.endswith("num_batches_tracked") or name in ("tokenizer.window", "tokenizer._mel_filters"):
-            return None
-        for src, dst in (
-            ("flow.encoder.",           "a.gen.fenc."),
-            ("flow.decoder.estimator.", "a.gen.est."),
-            ("mel2wav.",                "a.gen.hift."),
-            ("speaker_encoder.",        "a.spk."),
-            ("tokenizer.",              "a.s3tok."),
-            ("flow.",                   "a.gen.flow."),  # input_embedding, encoder_proj, spk_embed_affine_layer
-            ("lstm.",                   "a.ve.lstm."),
-            ("proj.",                   "a.ve.proj."),
-            ("cond_enc.",               "a.cenc."),
-        ):
-            if name.startswith(src):
-                new_name = dst + name[len(src):]
-                break
-        if name == "text_pos_emb.emb.weight":
-            new_name = "a.gen.t3.text_pos_emb"
-        elif name == "speech_pos_emb.emb.weight":
-            new_name = "a.gen.t3.speech_pos_emb"
-        elif name == "speech_emb.weight":
-            new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD] + ".weight"
-        if new_name is None:
-            return None
-        return super().filter_tensors((new_name, gen))
-
-    def set_gguf_parameters(self):
-        self.gguf_writer.add_file_type(self.ftype)
-
-        # speaker encoder, the front-end hparams are hard-coded in clip.cpp
-        self.gguf_writer.add_clip_has_audio_encoder(True)
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX_SPKENC)
-        self.gguf_writer.add_audio_projection_dim(80)
-        self.gguf_writer.add_audio_num_mel_bins(80)
-        self.gguf_writer.add_audio_block_count(0)
-        self.gguf_writer.add_audio_embedding_length(192)
-        self.gguf_writer.add_audio_head_count(1)
-        self.gguf_writer.add_audio_feed_forward_length(192)
-        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-
-        # audio generator (s3gen, chatterbox projector); the flow encoder shape
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX)
-        self.gguf_writer.add_gen_audio_projection_dim(80)
-        self.gguf_writer.add_gen_audio_embedding_length(512)
-        self.gguf_writer.add_gen_audio_feed_forward_length(2048)
-        self.gguf_writer.add_gen_audio_block_count(6)
-        self.gguf_writer.add_gen_audio_head_count(8)
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
-
-    def _fuse_weight_norm(self, name: str, data_torch: Tensor) -> tuple[str, Tensor] | None:
-        # torch weight_norm parametrization: weight = g * v / |v| over dims 1..n
-        base = name.split(".parametrizations.weight.original")[0]
-        if name.endswith("original0"):
-            self._wnorm_g[base] = data_torch
-        else:
-            self._wnorm_v[base] = data_torch
-        if base in self._wnorm_g and base in self._wnorm_v:
-            g = self._wnorm_g.pop(base)
-            v = self._wnorm_v.pop(base)
-            norm = v.float().norm(dim=tuple(range(1, v.dim())), keepdim=True)
-            return (base + ".weight", g.float() * v.float() / norm)
-        return None
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if ".parametrizations.weight.original" in name:
-            fused = self._fuse_weight_norm(name, data_torch)
-            if fused is not None:
-                yield fused
-            return
-        yield (name, data_torch)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        del name, bid
-        # tensors read on the host stay F32
-        if new_name.startswith(("a.ve.", "a.cenc.", "voice.", "a.gen.t3.", "a.gen.hift.m_source.")):
-            return gguf.GGMLQuantizationType.F32
-        # the conv ops have no BF16 kernels
-        if n_dims >= 2 and new_name.endswith((".weight", ".weight_v")):
-            return gguf.GGMLQuantizationType.F16
-        return gguf.GGMLQuantizationType.F32
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        conds = torch.load(self.dir_model / "conds.pt", map_location="cpu", weights_only=True)
-        t3c = conds["t3"] if isinstance(conds, dict) else conds.t3
-        genc = conds["gen"] if isinstance(conds, dict) else conds.gen
-        t3c = vars(t3c) if not isinstance(t3c, dict) else t3c
-        genc = vars(genc) if not isinstance(genc, dict) else genc
-
-        def talker_tensor(name: str) -> Tensor:
-            return LazyTorchTensor.to_eager(self.model_tensors[name]())
-
-        # the conds.pt voice: voice.default holds the talker conditioning rows, voice.default.ref_* the flow decoder reference
-        # 1D tensors are written as F32, the speech ids stay exact
-        yield ("voice.default.ref_tokens", genc["prompt_token"][0].float())
-        yield ("voice.default.ref_feat", genc["prompt_feat"][0].float())
-        yield ("voice.default.ref_spk", genc["embedding"][0].float())
-
-        spkr_w = talker_tensor("a.cenc.spkr_enc.weight").float()
-        spkr_b = talker_tensor("a.cenc.spkr_enc.bias").float()
-        spkr_row = spkr_w @ t3c["speaker_emb"][0].float() + spkr_b
-        speech_emb = talker_tensor(self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD)).float()
-        cond_tokens = t3c["cond_prompt_speech_tokens"][0]
-
-        if self.is_turbo:
-            # turbo talker conditioning: [spkr, speech rows of the cond tokens]
-            yield ("voice.default", torch.cat([spkr_row[None], speech_emb[cond_tokens]], dim=0))
-            return
-
-        # multilingual talker conditioning [spkr, perceiver x32, emotion], from the reference perceiver over the cond speech tokens
-        with torch.no_grad():
-            speech_pos = talker_tensor("a.gen.t3.speech_pos_emb").float()
-            pse = speech_emb[cond_tokens] + speech_pos[: cond_tokens.shape[0]]
-
-            ln_w = talker_tensor("a.cenc.perceiver.attn.norm.weight").float()
-            ln_b = talker_tensor("a.cenc.perceiver.attn.norm.bias").float()
-            wq = talker_tensor("a.cenc.perceiver.attn.to_q.weight").float()
-            bq = talker_tensor("a.cenc.perceiver.attn.to_q.bias").float()
-            wk = talker_tensor("a.cenc.perceiver.attn.to_k.weight").float()
-            bk = talker_tensor("a.cenc.perceiver.attn.to_k.bias").float()
-            wv = talker_tensor("a.cenc.perceiver.attn.to_v.weight").float()
-            bv = talker_tensor("a.cenc.perceiver.attn.to_v.bias").float()
-            wo = talker_tensor("a.cenc.perceiver.attn.proj_out.weight").float()
-            bo = talker_tensor("a.cenc.perceiver.attn.proj_out.bias").float()
-            query = talker_tensor("a.cenc.perceiver.pre_attention_query")[0].float()
-
-            n_head = 4
-            n_e = query.shape[1]
-
-            def attn_block(x1: Tensor, x2: Tensor) -> Tensor:
-                nx1 = F.layer_norm(x1, (n_e,), ln_w, ln_b)
-                nx2 = F.layer_norm(x2, (n_e,), ln_w, ln_b)
-                q = (nx1 @ wq.T + bq).view(-1, n_head, n_e // n_head).transpose(0, 1)
-                k = (nx2 @ wk.T + bk).view(-1, n_head, n_e // n_head).transpose(0, 1)
-                v = (nx2 @ wv.T + bv).view(-1, n_head, n_e // n_head).transpose(0, 1)
-                ctx = F.scaled_dot_product_attention(q, k, v)
-                ctx = ctx.transpose(0, 1).reshape(-1, n_e)
-                return ctx @ wo.T + bo + x1
-
-            pre = attn_block(query, pse)
-            p32 = attn_block(pre, pre)
-
-            emo = talker_tensor("a.cenc.emotion_adv_fc.weight").float()
-            emo_row = emo[:, 0] * t3c["emotion_adv"].reshape(-1)[0]
-            yield ("voice.default", torch.cat([spkr_row[None], p32, emo_row[None]], dim=0))

conversion/clover.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-
-from typing import Callable, Iterable, TYPE_CHECKING
-
-from .base import ModelBase, gguf
-from .qwen import _LinearAttentionVReorderBase
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-
-@ModelBase.register("CloverMoeModelLM")
-@ModelBase.example("Diffuspace/Clover-35B-A3B-Preview")
-class CloverMoeModel(_LinearAttentionVReorderBase):
-    model_arch = gguf.MODEL_ARCH.CLOVER_MOE
-    supports_mtp_export = False
-
-    def __init__(self, *args, **kwargs):
-        self.no_mtp = True
-        super().__init__(*args, **kwargs)
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self.gguf_writer.add_expert_shared_count(self.hparams.get("num_shared_experts", 1))
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, _ = item
-        if name.endswith((".conv_right_weight", ".bd_softmax_logit_scale", ".bd_softmax_history_bias", ".bd_softmax_history_c")):
-            return None
-
-        return super().filter_tensors(item)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.endswith(".A_log") or (name.endswith("norm.weight") and not name.endswith("linear_attn.norm.weight")):
-            data_torch = data_torch.float()
-
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/deepseek.py

@@ -236,12 +236,6 @@ class DeepseekModel(TextModel):
 class DeepseekV2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.DEEPSEEK2
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        # MLA projections do not have runtime weight-scale support.
-        if name.endswith((".q_a_proj.weight", ".q_b_proj.weight", ".kv_a_proj_with_mqa.weight", ".kv_b_proj.weight")):
-            return ()
-        return super()._map_fp8_weight_names(name)
-
     # TODO @ngxson : remove this when we support MTP for deepseek models
     skip_mtp = True
 

conversion/drama3.py

@@ -1,251 +0,0 @@
-from __future__ import annotations
-
-from typing import Any, Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, MmprojModel, gguf
-from .qwen import Qwen3_5MoeTextModel
-
-# Fish Audio Drama 3 (FishOmniModel)
-# text model: Qwen3.5-MoE backbone, samples the main token of each frame
-# mmproj: codebook tower and Fish DAC decoder (codec.pth)
-
-_TOWER_SHARED = "audio_decoder_mtp_shared.layers."
-_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
-
-
-@ModelBase.register("FishOmniModel")
-@ModelBase.example("fishaudio/drama-3")
-class Drama3TextModel(Qwen3_5MoeTextModel):
-    model_arch = gguf.MODEL_ARCH.QWEN35MOE
-    no_mtp = True  # the checkpoint has no MTP head
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        if item[0].startswith("audio_decoder"):
-            return None
-        return super().filter_tensors(item)
-
-
-@ModelBase.register("FishOmniModel")
-@ModelBase.example("fishaudio/drama-3")
-class Drama3MmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = False
-
-    _LAYER_MAP = {
-        "enorm.weight":                                 (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
-        "hnorm.weight":                                 (gguf.MODEL_TENSOR.A_GEN_CODE_HNORM, True),
-        "eh_proj.weight":                               (gguf.MODEL_TENSOR.A_GEN_CODE_EH_PROJ, False),
-        "semantic_embedding.weight":                    (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
-        "codebook_embedding.weight":                    (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_EMBD, False),
-        "final_layernorm.weight":                       (gguf.MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM, True),
-        "transformer_layer.attention_norm.weight":      (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, True),
-        "transformer_layer.ffn_norm.weight":            (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, True),
-        "transformer_layer.attention.q_norm.weight":    (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, True),
-        "transformer_layer.attention.k_norm.weight":    (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, True),
-        "transformer_layer.attention.wo.weight":        (gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, False),
-        "transformer_layer.feed_forward.w1.weight":     (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, False),
-        "transformer_layer.feed_forward.w3.weight":     (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, False),
-        "transformer_layer.feed_forward.w2.weight":     (gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, False),
-    }
-
-    _heads: dict[int, Tensor]
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._heads = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        cfg = dict(self.global_config["audio_decoder_config"])
-        cfg["num_hidden_layers"] = cfg["shared_enrichment_layers"] + cfg["num_codebooks"] - 1
-        return cfg
-
-    def set_gguf_parameters(self):
-        cfg = self.hparams
-        self.gguf_writer.add_file_type(self.ftype)
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
-        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
-        self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
-        self.gguf_writer.add_gen_audio_block_count(cfg["num_hidden_layers"])
-        self.gguf_writer.add_gen_audio_head_count(cfg["n_head"])
-        self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"])
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"])
-        # other tower and codec hparams are hardcoded in clip.cpp
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name = item[0]
-        if name.startswith("audio_decoder") or name == "model.language_model.embed_tokens.weight":
-            return item
-        return None
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
-        if new_name.endswith(".weight") and (
-            new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight")
-            or (".up.blk." in new_name and new_name.endswith(".dwconv.weight"))
-            or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
-        ):
-            return gguf.GGMLQuantizationType.F16
-        # keep ConvTranspose1d kernels and folded codebook tables in F32
-        if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
-            return gguf.GGMLQuantizationType.F32
-        if new_name.startswith("a.gen.wav.quant."):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        T = gguf.MODEL_TENSOR
-        cfg = self.hparams
-
-        if name.startswith("a.gen.wav."):
-            yield (name, data_torch)
-            return
-
-        if name == "model.language_model.embed_tokens.weight":
-            # semantic band of the backbone embeddings
-            begin = self.global_config["semantic_start_token_id"]
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
-            return
-
-        if name == "audio_decoder.codebook_embeddings.weight":
-            n_cb = cfg["num_codebooks"]
-            yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(n_cb, -1, data_torch.shape[-1]))
-            return
-
-        if name.startswith("audio_decoder_mtp.heads."):
-            n_heads = cfg["num_codebooks"] - 1
-            self._heads[int(name.split(".")[2])] = data_torch
-            if len(self._heads) < n_heads:
-                return
-            yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), torch.stack([self._heads.pop(i) for i in range(n_heads)]))
-            return
-
-        for prefix, offset in ((_TOWER_SHARED, 0), (_TOWER_RESIDUAL, cfg["shared_enrichment_layers"])):
-            if not name.startswith(prefix):
-                continue
-            idx, key = name[len(prefix):].split(".", 1)
-            il = int(idx) + offset
-            if key == "transformer_layer.attention.wqkv.weight":
-                # rows are [query | gate | key | value]
-                n_q = cfg["n_head"] * cfg["head_dim"]
-                n_kv = cfg["n_local_heads"] * cfg["head_dim"]
-                q, gate, k, v = data_torch.split([n_q, n_q, n_kv, n_kv], dim=0)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, il), q)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_GATE, il), gate)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, il), k)
-                yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, il), v)
-                return
-            tensor, zero_centered = self._LAYER_MAP[key]
-            if zero_centered:
-                data_torch = data_torch.float() + 1
-            yield (self.format_tensor_name(tensor, il), data_torch)
-            return
-
-        raise ValueError(f"unexpected tensor {name}")
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # only the DAC decoder, the encoder is for voice cloning (not supported yet)
-        sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
-        sd = sd.get("state_dict", sd)
-        T = gguf.MODEL_TENSOR
-
-        def get(name: str) -> Tensor:
-            return sd[name].float()
-
-        def conv(prefix: str) -> Tensor:
-            # fold weight norm
-            if prefix + ".conv.weight" in sd:
-                return get(prefix + ".conv.weight")
-            g = get(prefix + ".conv.parametrizations.weight.original0")
-            v = get(prefix + ".conv.parametrizations.weight.original1")
-            return g * v / v.norm(dim=(1, 2), keepdim=True)
-
-        def quant_table(prefix: str) -> Tensor:
-            # fold out_proj into the codebook, so decoding is a row lookup
-            g = get(prefix + ".out_proj.weight_g")
-            v = get(prefix + ".out_proj.weight_v")
-            w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
-            return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
-
-        def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
-            alpha = alpha.flatten()
-            return alpha, 1.0 / (alpha + 1e-9)
-
-        q = "quantizer."
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
-        n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB),
-               torch.stack([quant_table(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]))
-
-        # post_module uses interleaved RoPE, permute q/k rows so NEOX RoPE gives the same result
-        tfm = q + "post_module."
-        n_head, d_head = 16, 64
-        order = torch.cat([torch.arange(0, d_head, 2), torch.arange(1, d_head, 2)])
-
-        def neox_rows(w: Tensor) -> Tensor:
-            return w.reshape(n_head, d_head, -1)[:, order].reshape(w.shape)
-
-        n_tfm = sum(1 for k in sd if k.startswith(tfm + "layers.") and k.endswith(".ffn_norm.weight"))
-        for il in range(n_tfm):
-            p = f"{tfm}layers.{il}."
-            wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_Q, il), neox_rows(wq))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_K, il), neox_rows(wk))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_V, il), wv)
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_OUT, il), get(p + "attention.wo.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_NORM, il), get(p + "attention_norm.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_ATTN_SCALE, il), get(p + "attention_layer_scale.gamma"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_NORM, il), get(p + "ffn_norm.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_GATE, il), get(p + "feed_forward.w1.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_UP, il), get(p + "feed_forward.w3.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_DOWN, il), get(p + "feed_forward.w2.weight"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_FFN_SCALE, il), get(p + "ffn_layer_scale.gamma"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get(tfm + "norm.weight"))
-
-        # upsample: 2x (ConvTranspose1d + ConvNeXt)
-        for il in range(2):
-            p = f"{q}upsample.{il}."
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".weight"), conv(p + "0"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_CONV, il, suffix=".bias"), get(p + "0.conv.bias"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".weight"), conv(p + "1.dwconv"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_DWCONV, il, suffix=".bias"), get(p + "1.dwconv.conv.bias"))
-            for key, tensor in (("norm", T.A_GEN_WAV_UP_NORM), ("pwconv1", T.A_GEN_WAV_UP_PW1), ("pwconv2", T.A_GEN_WAV_UP_PW2)):
-                yield (self.format_tensor_name(tensor, il, suffix=".weight"), get(f"{p}1.{key}.weight"))
-                yield (self.format_tensor_name(tensor, il, suffix=".bias"), get(f"{p}1.{key}.bias"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_UP_GAMMA, il, suffix=""), get(p + "1.gamma"))
-
-        # DAC: entry conv, 4x (Snake, ConvTranspose1d, 3 residual units), Snake, post conv
-        d = "decoder.model."
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), conv(d + "0"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
-        for bid in range(4):
-            p = f"{d}{bid + 1}.block."
-            a, b = snake(get(p + "0.alpha"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
-            yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(p + "1.conv.bias"))
-            for xid in range(3):
-                r = f"{p}{xid + 2}.block."
-                for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
-                    a, b = snake(get(r + act + ".alpha"))
-                    name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
-                    yield (name + ".alpha", a)
-                    yield (name + ".beta", b)
-                for cv, tensor in (("1", T.A_GEN_WAV_DAC_RES_CONV1), ("3", T.A_GEN_WAV_DAC_RES_CONV2)):
-                    name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
-                    yield (name + ".weight", conv(r + cv))
-                    yield (name + ".bias", get(r + cv + ".conv.bias"))
-        a, b = snake(get(d + "5.alpha"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
-        yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias"))

conversion/lfm2.py

@@ -15,30 +15,25 @@ from .base import MmprojModel, ModelBase, TextModel, gguf, jinja_str_or_json, lo
 from .gemma import ConformerAudioModel
 
 
-def lfm2_feed_forward_length(hparams: dict[str, Any]) -> int:
-    ff_dim = hparams.get("block_ff_dim", hparams.get("intermediate_size"))
-    assert ff_dim is not None
-    auto_adjust_ff_dim = hparams["block_auto_adjust_ff_dim"]
-    ffn_dim_multiplier = hparams["block_ffn_dim_multiplier"]
-    multiple_of = hparams["block_multiple_of"]
-
-    if auto_adjust_ff_dim:
-        ff_dim = int(2 * ff_dim / 3)
-        # custom dim factor multiplier
-        if ffn_dim_multiplier is not None:
-            ff_dim = int(ffn_dim_multiplier * ff_dim)
-        ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
-
-    return ff_dim
-
-
 @ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
 @ModelBase.example("LiquidAI/LFM2-1.2B", "LiquidAI/LFM2.5-350M")
 class LFM2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.LFM2
 
     def _add_feed_forward_length(self):
-        self.gguf_writer.add_feed_forward_length(lfm2_feed_forward_length(self.hparams))
+        ff_dim = self.find_hparam(["block_ff_dim", "intermediate_size"])
+        auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]
+        ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]
+        multiple_of = self.hparams["block_multiple_of"]
+
+        if auto_adjust_ff_dim:
+            ff_dim = int(2 * ff_dim / 3)
+            # custom dim factor multiplier
+            if ffn_dim_multiplier is not None:
+                ff_dim = int(ffn_dim_multiplier * ff_dim)
+            ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
+
+        self.gguf_writer.add_feed_forward_length(ff_dim)
 
     def set_gguf_parameters(self):
         # set num_key_value_heads only for attention layers
@@ -77,9 +72,7 @@ def _is_d1_checkpoint(dir_model: Path) -> bool:
     if not (dir_model / "config.json").is_file():
         return False
     with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    # d1 on LFM3 is converted by conversion/lfm3.py
-    return config.get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") and not config.get("model_type", "").startswith("lfm3")
+        return json.load(f).get("auto_map", {}).get("AutoModel", "").endswith(".D1Model")
 
 
 @ModelBase.register_hparams_loader(_is_d1_checkpoint)

conversion/lfm3.py

@@ -1,629 +0,0 @@
-from __future__ import annotations
-
-import json
-import math
-
-from pathlib import Path
-from typing import Any, Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import MmprojModel, ModelBase, TextModel, gguf, logger
-from .lfm2 import D1Model, lfm2_feed_forward_length
-
-
-@ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration", "Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-1.7B", "LiquidAI/LFM3-1.7B-Base")
-class LFM3Model(TextModel):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # LFM3-Audio/Omni: encoder, projector and audio-output heads live next to the text model
-    _media_prefixes = ("encoder.", "mlp.", "codebook_embedding.", "depth_embeddings.", "depth_linear.", "depthformer.",
-                       "mask_embedding", "model.codebook_embedding.", "model.depth_decoder.")
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-
-        if name.startswith(cls._media_prefixes):
-            return None
-
-        name = name.removeprefix("lfm.")
-
-        return super().filter_tensors((name, gen))
-
-    # LFM3-Audio live ASR prompt of the model card, the audio follows in the assistant turn
-    _asr_template = (
-        "{{- bos_token -}}"
-        "{{- '<|im_start|>system\\nPerform live ASR.<|im_end|>\\n' -}}"
-        "{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}"
-    )
-
-    # TTS prompt of the model card: a language code is looked up, anything else is taken as the language name
-    _tts_langs = {
-        "ar": "Arabic", "da": "Danish", "de": "German", "el": "Greek", "en": "American English", "en-us": "American English",
-        "en-gb": "British English", "fi": "Finnish", "fr": "French", "he": "Hebrew", "it": "Italian", "ja": "Japanese",
-        "ko": "Korean", "nl": "Dutch", "pl": "Polish", "pt": "Portuguese", "es": "Spanish", "sv": "Swedish", "tr": "Turkish",
-        "zh": "Chinese",
-    }
-
-    # "language" and "voice" are optional, the text to speak is the first message
-    def _tts_template(self, think: bool) -> str:
-        return (
-            "{%- set names = " + json.dumps(self._tts_langs) + " -%}"
-            "{%- set code = (language | lower) if language is defined and language else 'en' -%}"
-            "{%- set lang = names[code] if code in names else language -%}"
-            "{%- set v = (voice | lower) if voice is defined and voice and (voice | lower) in ['female', 'male'] else 'female' -%}"
-            "{{- bos_token -}}"
-            "{{- '<|im_start|>system\\nPerform TTS in ' + lang + ' using a ' + v + ' voice.<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>user\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
-            "{{- '<|im_start|>assistant\\n" + ("<think></think>" if think else "") + "<|audio_start|>' -}}"
-        )
-
-    def set_vocab(self):
-        super().set_vocab()
-        # hparams hold the merged text config, the checkpoint's own model_type is in config.json
-        with open(self.dir_model / "config.json", encoding="utf-8") as f:
-            model_type = json.load(f).get("model_type")
-        is_audio = model_type == "lfm3_audio"
-        is_omni  = model_type == "lfm3_omni"
-        templates = []
-        if is_audio:
-            templates.append({"name": "asr", "template": self._asr_template})
-        if is_audio or is_omni:
-            # LFM3-Omni opens every assistant turn with an empty think block
-            templates.append({"name": "tts", "template": self._tts_template(think=is_omni)})
-        if templates:
-            self.gguf_writer.add_chat_template(templates)
-
-    def set_gguf_parameters(self):
-        layer_types = self.hparams["layer_types"]
-        n_kv_heads = self.hparams["num_key_value_heads"]
-
-        # set num_key_value_heads only for attention layers
-        self.hparams["num_key_value_heads"] = [
-            n_kv_heads if lt != "conv" else 0
-            for lt in layer_types
-        ]
-
-        super().set_gguf_parameters()
-
-        self.gguf_writer.add_rope_dimension_count(self.hparams["head_dim"])
-
-        self.gguf_writer.add_sliding_window_pattern([lt == "sliding_attention" for lt in layer_types])
-        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
-
-        self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # Partial RoPE is represented by masking trailing frequency pairs.
-        # rope_parameters is keyed by layer type; only full_attention layers use partial RoPE.
-        rope_parameters = self.hparams["rope_parameters"]["full_attention"]
-        partial = rope_parameters.get("partial_rotary_factor", 1.0)
-        head_dim = self.hparams["head_dim"]
-        rope_angles = int(partial * head_dim) // 2
-        n_pairs = head_dim // 2
-        assert 0 < rope_angles <= n_pairs
-        values = [1.0] * rope_angles + [1e30] * (n_pairs - rope_angles)
-        rope_freqs = torch.tensor(values, dtype=torch.float32)
-        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # conv op requires 2d tensor
-        if "conv.conv" in name:
-            data_torch = data_torch.squeeze(1)
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-# d1 on LFM3-VL / LFM3-Omni: the stock weights, the d1 prompt read at its last token
-_D1_TYPES = {"lfm3_vl": gguf.DecisionType.LFM3_D1, "lfm3_omni": gguf.DecisionType.LFM3_D1_OMNI}
-
-
-def _is_lfm3_d1_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("architectures") == ["D1Model"] and config.get("model_type") in _D1_TYPES
-
-
-@ModelBase.register_hparams_loader(_is_lfm3_d1_checkpoint)
-def _load_lfm3_d1_hparams(dir_model: Path) -> dict[str, Any]:
-    logger.info("gguf: detected LFM3 d1 checkpoint")
-    hparams = ModelBase.load_hparams(dir_model, False, guess=False)
-    hparams["d1_family"] = hparams["model_type"]
-    # the mmproj is the base model's, the text model writes the decision
-    hparams["architectures"] = ["Lfm3VlForConditionalGeneration" if hparams["model_type"] == "lfm3_vl" else "Lfm3OmniForConditionalGeneration"]
-    hparams["text_config"]["architectures"] = ["Lfm3D1Model"]
-    return hparams
-
-
-@ModelBase.register("Lfm3D1Model")
-@ModelBase.example("LiquidAI/d1-2B", "LiquidAI/d1-omni-2B")
-class LFM3D1Model(LFM3Model):
-    model_arch = gguf.MODEL_ARCH.LFM3
-
-    # how the checkpoint serves, hard-coded in the server: assert the checkpoint records the same
-    _served = {"d1_codes": 2, "d1_attention": "set"}
-
-    def set_vocab(self):
-        # the vocab without the speech task templates of LFM3-Audio/Omni, d1 only decides
-        TextModel.set_vocab(self)
-        # the d1 prompt of LFM2, the LFM3 template writes an empty thought before the answer
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": D1Model._systemone_template() + "<think></think>"}])
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-
-        for key, value in self._served.items():
-            if self.hparams.get(key) != value:
-                raise ValueError(f"unsupported d1 checkpoint: {key} = {self.hparams.get(key)!r}, expected {value!r}")
-
-        # a lone question reads the whole prompt both ways on the full attention layers
-        self.gguf_writer.add_causal_attention(False)
-        self.gguf_writer.add_decision_type(_D1_TYPES[self.hparams["d1_family"]])
-
-        # the option answers share one temperature
-        with open(self.dir_model / "calibration.json", encoding="utf-8") as f:
-            calibration = json.load(f)
-        if calibration.get("temperatures"):
-            raise ValueError("unsupported d1 calibration: per-bucket temperatures")
-        for name in ("noul", "choice", "score"):
-            self.gguf_writer.add_decision_temperature(name, calibration["default"])
-
-
-class LFM3VLVisionMixin(MmprojModel):
-    """LFM3-VL vision tower and projector, shared with LFM3-Omni."""
-
-    def _set_lfm3vl_gguf_parameters(self):
-        assert self.hparams_vision is not None
-        # clip.cpp hardcodes the GELU projector and its input norm eps
-        if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
-                or self.global_config["projector_layer_norm_eps"] != 1e-6):
-            raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
-        # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
-        if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
-                or self.hparams_vision["conv_kernel_size"] != 3):
-            raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
-        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.LFM3VL)
-        self.gguf_writer.add_vision_head_count_kv(self.hparams_vision["num_key_value_heads"])
-        self.gguf_writer.add_vision_spatial_merge_size(self.global_config["downsample_factor"])
-        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["norm_eps"])
-        self.gguf_writer.add_vision_use_silu(True)
-        size = self.preprocessor_config["size"]
-        self.gguf_writer.add_vision_min_pixels(size["shortest_edge"])
-        self.gguf_writer.add_vision_max_pixels(size["longest_edge"])
-        # PIL resample enum, the torchvision backend has no lanczos
-        resample = self.preprocessor_config["resample"]
-        resize_algo = {2: "bilinear", 3: "bicubic"}.get(resample)
-        if resize_algo is None:
-            raise ValueError(f"unsupported resample: {resample}")
-        self.gguf_writer.add_vision_image_resize_algo(resize_algo)
-        self._check_video_config()
-
-    # mtmd_get_video_ts_style and the lfm3vl video frame sizing hardcode these, conversion fails on a mismatch
-    _video = {"fps": 2.0, "min_frames": 4, "temporal_patch_size": 2, "size": {"shortest_edge": 4096, "longest_edge": 25165824}}
-    _video_max_frames = 768
-
-    def _check_video_config(self):
-        import json
-        video = self.preprocessor_config.get("video_processor")
-        path = self.dir_model / "video_preprocessor_config.json"
-        if video is None and path.is_file():
-            with open(path, encoding="utf-8") as f:
-                video = json.load(f)
-        if video is None:
-            return
-        for k, v in {**self._video, "max_frames": self._video_max_frames}.items():
-            if video.get(k) != v:
-                raise ValueError(f"unsupported video {k}={video.get(k)!r}, expected {v!r}")
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if "patch_embd" in new_name:
-            return gguf.GGMLQuantizationType.F16
-        if "conv2d_spatial" in new_name:
-            # CUDA CONV_2D_DW requires an f32 kernel; the tensor is tiny
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def _lfm3vl_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "model.vision_tower.patchifier.weight":
-            # qwen2vl-style layout [channel, temporal, ph, pw], one conv kernel per temporal slice
-            assert self.hparams_vision is not None
-            t = self.hparams_vision["temporal_patch_size"]
-            p = self.hparams_vision["patch_size"]
-            n_embd = data_torch.shape[0]
-            w = data_torch.view(n_embd, 3, t, p, p)
-            yield ("v.patch_embd.weight", w[:, :, 0].contiguous())
-            if t == 2:
-                yield ("v.patch_embd.weight.1", w[:, :, 1].contiguous())
-            else:
-                assert t == 1, f"unsupported vision_temporal_patch_size {t}"
-            return
-        if name == "model.vision_tower.embedding_norm.weight":
-            yield ("v.embedding_norm.weight", data_torch)
-            return
-        if "conv.in_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_in_proj.weight", data_torch)
-            return
-        if "conv.spatial_conv.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_spatial.weight", data_torch)
-            return
-        if "conv.out_proj.weight" in name:
-            yield (f"v.blk.{bid}.conv2d_out_proj.weight", data_torch)
-            return
-
-        # projector: input LayerNorm -> linear_1 -> GELU -> linear_2
-        if name.startswith("model.multi_modal_projector."):
-            layer, suffix = name.removeprefix("model.multi_modal_projector.").split(".")
-            yield ({"layer_norm": "mm.input_norm", "linear_1": "mm.0", "linear_2": "mm.2"}[layer] + "." + suffix, data_torch)
-            return
-
-        yield (self.map_tensor_name(name), data_torch)
-
-
-@ModelBase.register("Lfm3VlForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-VL-2B")
-class LFM3VLMmprojModel(LFM3VLVisionMixin):
-    has_vision_encoder = True
-    has_audio_encoder = False
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-
-
-def _is_lfm3_audio_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    return config.get("model_type") == "lfm3_audio" and "lfm" in config
-
-
-# read the release config.json as is: a transformers that knows lfm3_audio rewrites it into another layout
-@ModelBase.register_hparams_loader(_is_lfm3_audio_checkpoint)
-def _load_lfm3_audio_hparams(dir_model: Path) -> dict[str, Any]:
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        config = json.load(f)
-    config["text_config"] = config["lfm"]
-    return config
-
-
-@ModelBase.register("Lfm3AudioForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Audio-2B")
-class LFM3AudioMmprojModel(MmprojModel):
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    # the downsampler has the LFM2-Audio pre-encoder layout: conv, dw conv, pw conv, dw conv, pw conv
-    _downsample_idx = {"conv_in": 0, "conv_mid": 2, "lin_mid": 3, "conv_end": 5, "lin_end": 6}
-
-    # audio output: MaskGIT depthformer (a.gen.code.*) and the codec decoder (a.gen.wav.*)
-    _depth_layer_map = {
-        "operator_norm":      gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
-        "self_attn.q_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q,
-        "self_attn.q_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,
-        "self_attn.k_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K,
-        "self_attn.k_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,
-        "self_attn.v_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V,
-        "self_attn.o_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
-        "self_attn.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
-        "ffn_norm":           gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
-        "feed_forward.w1":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
-        "feed_forward.w3":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
-        "feed_forward.w2":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
-    }
-    _codec_layer_map = {
-        "operator_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM,
-        "post_operator_norm":    gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM,
-        "self_attn.q_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q,
-        "self_attn.q_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM,
-        "self_attn.k_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K,
-        "self_attn.k_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM,
-        "self_attn.v_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V,
-        "self_attn.o_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT,
-        "self_attn.gate_proj":   gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE,
-        "conv.in_proj":          gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1,
-        "conv.conv":             gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW,
-        "conv.out_proj":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2,
-        "ffn_norm":              gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM,
-        "post_ffn_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM,
-        "feed_forward.w1":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE,
-        "feed_forward.w3":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
-        "feed_forward.w2":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
-    }
-    # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
-    _codec = {
-        "model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256,
-        "encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6,
-        "decoder_window": 30, "sampling_rate": 24000,
-    }
-    # the lfm3a pipeline in mtmd-helper-gen.cpp hardcodes these
-    _generation = {"codebooks": 12, "maskgit_schedule": [4, 4, 4], "interleaved_n_text": 6, "interleaved_n_audio": 12}
-
-    # speech-code sampling of the model card, generation_config.json overrides it
-    _audio_sampling = {
-        "audio_temperature": 1.0, "audio_top_k": 128, "audio_repetition_penalty": 1.1, "audio_repetition_window": 4096,
-    }
-
-    # mtmd_audio_preprocessor_lfm3a hardcodes this front end, conversion fails on a mismatch
-    _feature_extractor = {
-        "sampling_rate": 16000, "feature_size": 128, "n_fft": 400, "win_length": 400, "hop_length": 160,
-        "log_floor": -12.0, "normalization_offset": 4.0, "normalization_divisor": 4.0, "encoder_stride": 8,
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # audio output is exported when the checkpoint ships the codec
-        self._audio_output = (self.dir_model / "audio_tokenizer").is_dir()
-        gen = self.get_generation_config()
-        self._n_codebooks: int = gen["codebooks"]
-        self._audio_vocab: int = gen["audio_vocab_size"]
-        self._depth_embd: dict[int, Tensor] = {}
-        self._depth_norm: dict[int, Tensor] = {}
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["encoder"]
-        cfg = dict(enc["lfm"])
-        cfg["num_mel_bins"] = enc["dim_in"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        steps = c["maskgit_steps"]
-        return {
-            "codebooks": c["codebooks"], "audio_vocab_size": c["audio_vocab_size"],
-            "maskgit_schedule": [c["codebooks"] // steps] * steps,
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depthformer"]
-
-    def set_gguf_parameters(self):
-        assert self.hparams_audio is not None
-        fe = self.preprocessor_config["feature_extractor"]
-        if fe["feature_extractor_type"] not in ("Lfm3AudioFeatureExtractor", "Lfm3OmniFeatureExtractor"):
-            raise ValueError(f"unsupported feature_extractor_type {fe['feature_extractor_type']!r}")
-        for k, v in self._feature_extractor.items():
-            if fe[k] != v:
-                raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}")
-
-        # clip.cpp hardcodes the encoder rope base
-        if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported audio encoder rope_theta")
-
-        self.hparams_audio["intermediate_size"] = lfm2_feed_forward_length(self.hparams_audio)
-        super().set_gguf_parameters()
-        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.LFM3A)
-        self.gguf_writer.add_audio_head_count_kv(self.hparams_audio["num_key_value_heads"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio["norm_eps"])
-        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])
-        self.gguf_writer.add_audio_window_size(self.hparams_audio["sliding_window"])
-
-        if not self._audio_output:
-            return
-
-        # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp
-        gen = self.get_generation_config()
-        for k, v in self._generation.items():
-            if gen[k] != v:
-                raise ValueError(f"unsupported {k}={gen[k]!r}, expected {v!r}")
-        depth = self.get_depth_config()
-        if depth.get("is_causal", False) or depth["use_sandwich_norm"] or set(depth["layer_types"]) != {"full_attention"}:
-            raise ValueError("unsupported depthformer, expected a bidirectional all-attention stack")
-        if depth["rope_parameters"]["full_attention"]["rope_theta"] != 1e6:
-            raise ValueError("unsupported depthformer rope_theta")
-        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
-        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.LFM3A_GEN)
-        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
-        self.gguf_writer.add_gen_audio_embedding_length(depth["hidden_size"])
-        self.gguf_writer.add_gen_audio_feed_forward_length(depth["intermediate_size"])
-        self.gguf_writer.add_gen_audio_block_count(depth["num_hidden_layers"])
-        self.gguf_writer.add_gen_audio_head_count(depth["num_attention_heads"])
-        self.gguf_writer.add_gen_audio_head_count_kv(depth["num_key_value_heads"])
-        self.gguf_writer.add_gen_audio_attention_layernorm_eps(depth["norm_eps"])
-
-        # sampling defaults of the speech codes, next to the text ones that become general.sampling.*
-        gen = gguf.Metadata.load_generation_config(self.dir_model)
-        for key, add in (("audio_temperature",        self.gguf_writer.add_gen_audio_sampling_temp),
-                         ("audio_top_k",              self.gguf_writer.add_gen_audio_sampling_top_k),
-                         ("audio_repetition_penalty", self.gguf_writer.add_gen_audio_sampling_penalty_repeat),
-                         ("audio_repetition_window",  self.gguf_writer.add_gen_audio_sampling_penalty_last_n)):
-            value = gen.get(key, self._audio_sampling.get(key))
-            if value is not None:
-                add(value)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if new_name.startswith("a.conv1d.") or "conv_dw" in new_name or new_name.startswith("a.gen.wav.quant."):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        if self._audio_output:
-            yield from self._codec_tensors()
-
-    def _codec_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        import json
-        from safetensors.torch import load_file
-
-        T = gguf.MODEL_TENSOR
-        codec_dir = self.dir_model / "audio_tokenizer"
-        with open(codec_dir / "config.json", encoding="utf-8") as f:
-            cfg = json.load(f)
-        for k, v in self._codec.items():
-            if cfg.get(k) != v:
-                raise ValueError(f"unsupported audio_tokenizer {k}={cfg.get(k)!r}, expected {v!r}")
-        if math.prod(cfg["fsq_levels"]) != self._audio_vocab - 1:
-            raise ValueError(f"audio_vocab_size {self._audio_vocab} does not match the codec, expected codebook size + 1")
-
-        state_dict = load_file(codec_dir / "model.safetensors")
-
-        # FSQ code table: index -> the len(levels) quantized values, see FiniteScalarQuantization._indices_to_codes
-        levels = torch.tensor(cfg["fsq_levels"], dtype=torch.int64)
-        basis = torch.cumprod(torch.cat([torch.ones(1, dtype=torch.int64), levels[:-1]]), dim=0)
-        half = levels // 2
-        idx = torch.arange(int(levels.prod()), dtype=torch.int64)[:, None]
-        codes = ((idx // basis) % levels - half) / half
-        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), codes.to(torch.float32))
-
-        # one projection over the concatenated code vectors of all codebooks, under quantizer.fsq. in newer exports
-        quantizer = "quantizer.fsq." if "quantizer.fsq.project_out.weight" in state_dict else "quantizer."
-        for suffix in ("weight", "bias"):
-            yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT, suffix=f".{suffix}"), state_dict[f"{quantizer}project_out.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=f".{suffix}"), state_dict[f"from_quantizer.{suffix}"])
-            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=f".{suffix}"), state_dict[f"decoder.lin.{suffix}"])
-        yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), state_dict["decoder.lfm.embedding_norm.weight"])
-
-        for name, data_torch in state_dict.items():
-            if not name.startswith("decoder.lfm.layers."):
-                continue
-            bid_str, key = name.removeprefix("decoder.lfm.layers.").rsplit(".weight", 1)[0].split(".", 1)
-            if key == "conv.conv":
-                data_torch = data_torch.squeeze(1)
-            yield (self.format_tensor_name(self._codec_layer_map[key], int(bid_str)), data_torch)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith("a.gen.wav."):
-            yield (name, data_torch)
-            return
-
-        if name.startswith(("depthformer.", "depth_", "codebook_embedding.", "mask_embedding")):
-            if self._audio_output:
-                yield from self._depthformer_tensor(data_torch, name, bid)
-            return
-
-        if not name.startswith(("encoder.", "mlp.")):
-            return
-
-        if name.startswith("encoder.downsample."):
-            layer, suffix = name.removeprefix("encoder.downsample.").split(".")
-            if layer == "lin_out":
-                if suffix == "weight":
-                    data_torch = data_torch.squeeze(-1)
-                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_OUT, suffix=f".{suffix}"), data_torch)
-                return
-            if suffix == "bias":
-                data_torch = data_torch[:, None, None]
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_CONV1D, self._downsample_idx[layer], f".{suffix}"), data_torch)
-            return
-
-        # conv op requires 2d tensor
-        if name.endswith("conv.conv.weight"):
-            data_torch = data_torch.squeeze(1)
-
-        yield (self.map_tensor_name(name), data_torch)
-
-    def _depthformer_tensor(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        T = gguf.MODEL_TENSOR
-
-        if name.startswith("depthformer.model.layers."):
-            key = name.removeprefix("depthformer.model.layers.").split(".", 1)[1].rsplit(".weight", 1)[0]
-            yield (self.format_tensor_name(self._depth_layer_map[key], bid), data_torch)
-            return
-        if name == "depthformer.model.embedding_norm.weight":
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
-            return
-        if name.startswith("depth_linear."):
-            yield (self.format_tensor_name(T.A_GEN_CODE_PROJ_IN, suffix="." + name.rsplit(".", 1)[1]), data_torch)
-            return
-        if name == "mask_embedding":
-            yield (self.format_tensor_name(T.A_GEN_CODE_MASK_EMBD, suffix=""), data_torch)
-            return
-        if name == "codebook_embedding.weight":
-            assert data_torch.shape[0] == self._n_codebooks * self._audio_vocab
-            yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch)
-            return
-
-        # per-codebook heads: the embedding table is tied to to_logits, merge the 12 tables into one tensor
-        assert name.startswith("depth_embeddings.")
-        idx = int(name.split(".")[1])
-        key = name.split(".", 2)[2]
-        if key == "embedding.weight":
-            self._depth_embd[idx] = data_torch
-            if len(self._depth_embd) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), torch.stack([self._depth_embd[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "embedding_norm.weight":
-            self._depth_norm[idx] = data_torch
-            if len(self._depth_norm) == self._n_codebooks:
-                yield (self.format_tensor_name(T.A_GEN_CODE_HEAD_NORM), torch.stack([self._depth_norm[i] for i in range(self._n_codebooks)], dim=0))
-        elif key == "to_logits.weight":
-            if not torch.equal(data_torch, self._depth_embd[idx]):
-                raise ValueError(f"depth_embeddings.{idx} is not tied")
-        else:
-            raise ValueError(f"unexpected tensor {name}")
-
-
-@ModelBase.register("Lfm3OmniForConditionalGeneration")
-@ModelBase.example("LiquidAI/LFM3-Omni-2B")
-class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel):
-    has_vision_encoder = True
-    has_audio_encoder = True
-
-    _audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64}
-    _video_max_frames = 384
-
-    # the audio side is the LFM3-Audio model under the HF-native names
-    _audio_renames = (
-        ("model.audio_encoder.lfm.", "encoder.lfm.lfm."),
-        ("model.audio_encoder.", "encoder."),
-        ("model.audio_projector.layer_norm.", "mlp.model.0."),
-        ("model.audio_projector.linear_1.", "mlp.model.1."),
-        ("model.audio_projector.linear_2.", "mlp.model.3."),
-        ("model.depth_decoder.model.", "depthformer.model."),
-        ("model.depth_decoder.", ""),
-        ("model.codebook_embedding.", "codebook_embedding."),
-    )
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        assert self.hparams_vision is not None
-        # dynamic resolution, image_size is only set for MmprojModel compatibility
-        self.hparams_vision["image_size"] = 512
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        enc = self.global_config["audio_encoder_config"]
-        cfg = dict(enc["text_config"])
-        cfg["num_mel_bins"] = enc["num_mel_bins"]
-        cfg["sliding_window"] = enc["sliding_window"]
-        return cfg
-
-    def get_generation_config(self) -> dict[str, Any]:
-        c = self.global_config
-        return {
-            "codebooks": c["num_codebooks"], "audio_vocab_size": c["codebook_size"], "maskgit_schedule": c["maskgit_schedule"],
-            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
-        }
-
-    def get_depth_config(self) -> dict[str, Any]:
-        return self.global_config["depth_config"]
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self._set_lfm3vl_gguf_parameters()
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
-            yield from self._lfm3vl_tensors(data_torch, name, bid)
-            return
-        for old, new in self._audio_renames:
-            if name.startswith(old):
-                yield from super().modify_tensors(data_torch, new + name.removeprefix(old), bid)
-                return
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/limite.py

@@ -1,184 +0,0 @@
-from __future__ import annotations
-
-from typing import Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, TextModel, gguf
-
-
-@ModelBase.register("LimiteForCausalLM")
-@ModelBase.example("paradigma-inc/limite-1b-violetto")
-class LimiteModel(TextModel):
-    model_arch = gguf.MODEL_ARCH.LIMITE
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hp = self.hparams
-
-        expected = {
-            "pos_mode": "rope",
-            "qk_norm": "rms_pre_rope",
-            "rope_style": "interleaved_pairs_odd_lane_sign_flip",
-            "rope_per_layer": False,
-            "rms_norm_has_weight": False,
-            "rms_norm_eps_mode": "torch_finfo_default",
-            "mlp_type": "swiglu",
-            "attn_gate_applied": "per_head_before_o_proj",
-            "ve_head_slice": "first_num_key_value_heads",
-            "ve_applied_before_qk_norm": True,
-            "global_nope": True,
-            "global_window": -1,
-            "mudd_r_site": "resid",
-            "tie_word_embeddings": True,
-            "final_softcap": 0.0,
-        }
-        for key, value in expected.items():
-            if hp.get(key) != value:
-                raise NotImplementedError(f"Limite: unsupported {key}={hp.get(key)!r}, expected {value!r}")
-        if hp["softcap_logits"]["kind"] != "sigmoid":
-            raise NotImplementedError(f"Limite: unsupported softcap_logits kind {hp['softcap_logits']['kind']!r}")
-
-        self.n_head    = hp["num_attention_heads"]
-        self.n_head_kv = hp["num_key_value_heads"]
-        self.head_dim  = hp["head_dim"]
-        self.n_pairs   = hp["rope_n_pairs"]
-
-        self.mudd_taps: dict[int, list[int]] = {}
-        if hp.get("mudd"):
-            if not hp.get("mudd_mlp"):
-                raise NotImplementedError("Limite: mudd without mudd_mlp is not supported")
-            self.mudd_taps = {int(il): [int(t) for t in taps] for il, taps in hp["mudd_tap_idx"].items()}
-            assert all(len(taps) == hp["mudd_taps"] for taps in self.mudd_taps.values())
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        hp = self.hparams
-
-        self.gguf_writer.add_key_length(self.head_dim)
-        self.gguf_writer.add_value_length(self.head_dim)
-
-        # torch rms_norm with eps=None uses the fp32 epsilon, even for bf16 inputs
-        self.gguf_writer.add_layer_norm_rms_eps(torch.finfo(torch.float32).eps)
-        self.gguf_writer.add_attention_scale(hp["attention_softmax_scale"])
-        self.gguf_writer.add_xsa_eps(hp["xsa_normalize_eps"] if hp.get("xsa") else 0.0)
-
-        # HF freq_i = base^(-i/(n_pairs-1)), ggml uses freq_base^(-i/n_pairs)
-        self.gguf_writer.add_rope_dimension_count(2 * self.n_pairs)
-        self.gguf_writer.add_rope_freq_base(hp["rope_base_local"] ** (self.n_pairs / (self.n_pairs - 1)))
-
-        # HF keeps keys k >= q - window, so a window of W spans W + 1 tokens
-        global_layers = set(hp["global_layers"])
-        self.gguf_writer.add_sliding_window(hp["sliding_window"] + 1)
-        self.gguf_writer.add_sliding_window_pattern([il not in global_layers for il in range(self.block_count)])
-
-        cap = hp["softcap_logits"]
-        self.gguf_writer.add_final_logit_sigmoid_capping(cap["a"], cap["b"], cap["c"])
-
-        if self.mudd_taps:
-            n_taps = hp["mudd_taps"]
-            tap_indices = [-1] * (self.block_count * n_taps)
-            for il, taps in self.mudd_taps.items():
-                tap_indices[il * n_taps:(il + 1) * n_taps] = taps
-            self.gguf_writer.add_mudd_feed_forward_length(hp["mudd_inter"])
-            self.gguf_writer.add_mudd_tap_count(n_taps)
-            self.gguf_writer.add_mudd_tap_indices(tap_indices)
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        # tiny mixing weights, kept at full precision as in the reference
-        # the mudd tensors come from a global source tensor, so take the block id from the new name
-        if new_name.startswith("blk."):
-            bid = int(new_name.split(".")[1])
-        for key in (gguf.MODEL_TENSOR.ATTN_GATE, gguf.MODEL_TENSOR.ATTN_VE_GATE, gguf.MODEL_TENSOR.MUDD_DOWN, gguf.MODEL_TENSOR.MUDD_UP):
-            if self.match_model_tensor_name(new_name, key, bid):
-                return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def _swap_rope_pairs(self, w: Tensor, n_head: int) -> Tensor:
-        # HF rotates each pair by -theta; swapping the pair lanes gives the ggml +theta rotation.
-        # q and k are both swapped, so the attention scores do not change.
-        idx = torch.arange(self.head_dim)
-        idx[:2 * self.n_pairs] = idx[:2 * self.n_pairs].view(-1, 2).flip(-1).reshape(-1)
-        w = w.reshape(n_head, self.head_dim, -1)
-        return w[:, idx, :].reshape(n_head * self.head_dim, -1)
-
-    def _pad_cols(self, w: Tensor) -> Tensor:
-        # gates read only the first channels of the input, pad them to n_embd
-        return torch.nn.functional.pad(w, (0, self.hparams["hidden_size"] - w.shape[-1]))
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        hp = self.hparams
-        T = gguf.MODEL_TENSOR
-
-        def load(n: str) -> Tensor:
-            return self.model_tensors[n]().float()
-
-        if name == "model.value_embeds.weight":
-            ve = data_torch.reshape(data_torch.shape[0], -1, hp["ve_dim"])[:, :self.n_head_kv]
-            ve = torch.nn.functional.pad(ve, (0, self.head_dim - hp["ve_dim"]))
-            yield self.format_tensor_name(T.VALUE_EMBD), ve.reshape(ve.shape[0], -1) * hp["ve_gate_scale"]
-            return
-
-        if name == "model.mudd.dense1":
-            for il in self.mudd_taps:
-                yield self.format_tensor_name(T.MUDD_DOWN, il), data_torch
-            return
-        if name == "model.mudd.dense2":
-            # rows [0, n) mix the attention input, rows [n, 2n) mix the residual
-            dense2_mlp = load("model.mudd.dense2_mlp")
-            bias       = load("model.mudd.bias")
-            bias_mlp   = load("model.mudd.bias_mlp")
-            for il, taps in self.mudd_taps.items():
-                n = len(taps)
-                w = torch.cat([data_torch[il, :n], dense2_mlp[il, :n]], 0)
-                b = torch.cat([bias[il, :n], bias_mlp[il, :n]], 0)
-                yield self.format_tensor_name(T.MUDD_UP, il), w
-                yield self.format_tensor_name(T.MUDD_UP, il, ".bias"), b
-            return
-        if name.startswith("model.mudd."):
-            return
-
-        if bid is None:
-            yield from super().modify_tensors(data_torch, name, bid)
-            return
-
-        prefix = f"model.layers.{bid}."
-        name = name.removeprefix(prefix)
-
-        if name in ("self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"):
-            data_torch = data_torch * load(prefix + "self_attn.qkv_scale")
-            if name == "self_attn.q_proj.weight":
-                data_torch = self._swap_rope_pairs(data_torch, self.n_head)
-            elif name == "self_attn.k_proj.weight":
-                data_torch = self._swap_rope_pairs(data_torch, self.n_head_kv)
-        elif name == "self_attn.o_proj.weight":
-            data_torch = data_torch * load(prefix + "self_attn.o_scale") * load(prefix + "post_lambda_attn") * hp["attn_gate_scale"]
-        elif name == "mlp.down_proj.weight":
-            data_torch = data_torch * load(prefix + "post_lambda_mlp")
-        elif name == "self_attn.attn_gate":
-            yield self.format_tensor_name(T.ATTN_GATE, bid), self._pad_cols(data_torch)
-            return
-        elif name == "self_attn.ve_gate":
-            yield self.format_tensor_name(T.ATTN_VE_GATE, bid), self._pad_cols(data_torch[:self.n_head_kv])
-            return
-        elif name == "self_attn.xsa_alpha":
-            if hp.get("xsa") and bid in hp["xsa_layers"]:
-                yield self.format_tensor_name(T.ATTN_XSA_ALPHA, bid), torch.tanh(data_torch)
-            return
-        elif name == "resid_lambda_attn":
-            yield self.format_tensor_name(T.ATTN_RESID_SCALE, bid), data_torch.reshape(1)
-            return
-        elif name == "resid_lambda_mlp":
-            yield self.format_tensor_name(T.FFN_RESID_SCALE, bid), data_torch.reshape(1)
-            return
-        elif name in ("self_attn.qkv_scale", "self_attn.o_scale", "post_lambda_attn", "post_lambda_mlp"):
-            # folded into the projections above
-            return
-
-        yield from super().modify_tensors(data_torch, prefix + name, bid)

conversion/metis.py

@@ -1,204 +0,0 @@
-from __future__ import annotations
-
-import math
-from typing import Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import LazyTorchTensor, ModelBase, TextModel, gguf
-
-
-@ModelBase.register("Metis16ForCausalLM")
-class Metis16Model(TextModel):
-    model_arch = gguf.MODEL_ARCH.METIS16
-
-    # read by generate_extra_tensors() or not used at inference: dropped in modify_tensors()
-    _folded = {
-        "pass_embeddings", "lm_head.weight", "depth_budget.multiplier",
-        "depth_memory.pass_embeddings", "depth_memory.anchor_embeddings",
-        "depth_memory.state_write.bias", "depth_memory.metadata_write.bias",
-    }
-    _folded_layer = {
-        "mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits",
-        "mixer_connection.read_logits", "mixer_connection.write_logits",
-        "moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits",
-        "moe_connection.read_logits", "moe_connection.write_logits",
-        "moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier",
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hp = self.hparams
-        self.n_pass = hp["num_passes"]
-        self.attn_layers = list(hp["attention_layer_indices"])
-        self._generated: set[str] = set()
-        # one cache slot per pass and layer, while the tensors exist once per physical layer
-        self.block_count = self.n_pass * hp["num_hidden_layers"]
-
-    def set_vocab(self):
-        self._set_vocab_gpt2()
-        # turns end on the first eos id, so llama.cpp must stop on it too
-        eos = self.hparams.get("eos_token_id")
-        if isinstance(eos, list) and eos:
-            self.gguf_writer.add_eot_token_id(eos[0])
-
-    def set_gguf_parameters(self):
-        hp = self.hparams
-        super().set_gguf_parameters()
-
-        n_layer = hp["num_hidden_layers"]
-        self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)])
-
-        self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"])
-        self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"])
-        self.gguf_writer.add_expert_shared_count(1)
-        self.gguf_writer.add_moe_latent_size(hp["latent_dim"])
-
-        d_inner = hp["mamba_expand"] * hp["hidden_size"]
-        self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"])
-        self.gguf_writer.add_ssm_inner_size(d_inner)
-        self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"])
-        self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"])
-        self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"])
-        self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"])
-
-        self.gguf_writer.add_hyper_connection_count(hp["n_streams"])
-        self.gguf_writer.add_metis_pass_count(self.n_pass)
-        self.gguf_writer.add_metis_memory_length(hp["memory_dim"])
-        self.gguf_writer.add_metis_route_length(hp["route_feature_dim"])
-        self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"])
-        self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"])
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        # the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it
-        if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")):
-            name += ".bias"
-        elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates"))
-              or name in ("depth_memory.stream_gate", "stream_embeddings")):
-            name += ".weight"
-        elif name == "depth_memory.stream_gate_bias":
-            name = "depth_memory.stream_gate.bias"
-        return super().filter_tensors((name, gen))
-
-    def _load(self, name: str) -> Tensor:
-        return LazyTorchTensor.to_eager(self.model_tensors[name]())
-
-    def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]:
-        self._generated.add(name)
-        return name, data
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        hp = self.hparams
-        r = hp["route_feature_dim"]
-        n_stream = hp["n_streams"]
-        T = gguf.MODEL_TENSOR
-        passes = self._load("pass_embeddings")[: self.n_pass]
-
-        # the mixes depend only on the weights and the pass, so Sinkhorn runs once here
-        # the controller runs in the checkpoint dtype, as the reference does
-        for il in range(hp["num_hidden_layers"]):
-            for src, (mix, read, write) in (
-                ("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)),
-                ("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)),
-            ):
-                prefix = f"layers.{il}.{src}."
-                ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias")
-                mixes, reads, writes = [], [], []
-                for p in range(self.n_pass):
-                    delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float()
-                    m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream])
-                    log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream)
-                    for _ in range(hp["mhc_sinkhorn_iterations"]):
-                        log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True)
-                        log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True)
-                    mixes.append(log_m.exp())
-                    reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1))
-                    writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1))
-                act = ctrl_w.dtype
-                yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float())
-                yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float())
-                yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float())
-
-        # one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width
-        meta_w = self._load("depth_memory.metadata_write.weight").float()
-        mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"]
-        base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float()
-                + mean_k * meta_w[:, r])
-        pass_e = self._load("depth_memory.pass_embeddings").float()
-        anchor_e = self._load("depth_memory.anchor_embeddings").float()
-        n_anchor = len(self.attn_layers) + 1
-        entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)])
-        yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias)
-
-    def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        # the graph reads the concatenated features one block at a time; each block is its own tensor
-        for i, part in enumerate(parts):
-            piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:]
-            yield from super().modify_tensors(piece, template.format(part), bid)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name in self._generated:
-            yield name, data_torch
-            return
-        if name in self._folded:
-            return
-        hp = self.hparams
-        d = hp["hidden_size"]
-
-        if bid is None:
-            if name == "stream_embeddings.weight":
-                # the graph keeps the streams innermost: [n_stream, n_embd, n_tokens]
-                yield from super().modify_tensors(data_torch.T.contiguous(), name, bid)
-                return
-            if name == "depth_memory.metadata_write.weight":
-                # metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized
-                r = hp["route_feature_dim"]
-                meta_w = data_torch.float()
-                entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"])
-                data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1)
-            elif name == "depth_memory.stream_gate.weight":
-                data_torch = data_torch.float() / math.sqrt(d)
-            elif name == "depth_memory.route_projection.weight":
-                yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid)
-                return
-            elif name == "continuation.hidden.weight":
-                yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid)
-                return
-            yield from super().modify_tensors(data_torch, name, bid)
-            return
-
-        local = name.removeprefix(f"layers.{bid}.")
-        if local in self._folded_layer:
-            return
-
-        if local == "mixer.impl.mixer.A_log":
-            data_torch = -torch.exp(data_torch).unsqueeze(-1)
-        elif local == "mixer.impl.mixer.D":
-            data_torch = data_torch.unsqueeze(-1)
-        elif local == "mixer.impl.mixer.conv1d.weight":
-            data_torch = data_torch.squeeze(1)
-        elif local == "mixer.impl.mixer.norm.weight":
-            data_torch = data_torch.reshape(hp["mamba_n_groups"], -1)
-        elif local == "mixer.pass_gates.weight":
-            # the reference applies tanh(gate) in the checkpoint dtype
-            data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float()
-        elif local == "moe.shared_expert.gate_up.weight":
-            gate, up = data_torch.chunk(2, dim=0)
-            yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid)
-            yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        # FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32
-        if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate"))
-                or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight",
-                                      "hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)

conversion/mistral4.py

@@ -1,217 +0,0 @@
-from __future__ import annotations
-
-from typing import Callable, Iterable, TYPE_CHECKING
-
-import torch
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, gguf
-
-from .mistral import MistralModel, MistralMoeModel
-
-
-@ModelBase.register("MistralLarge4ForConditionalGeneration")
-class MistralLarge4Model(MistralMoeModel):
-    model_arch = gguf.MODEL_ARCH.MISTRAL_LARGE4
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        n_layer = self.hparams["num_hidden_layers"]
-        dsa = self.hparams["dsa"]
-
-        def expand(pattern: str) -> list[str]:
-            values = pattern.split(",")
-            return [values[i % len(values)] for i in range(n_layer)]
-
-        self.layer_has_rope = [v == "rope" for v in expand(self.hparams.get("nope") or "rope")]
-        self.layer_indexer_full = [v == "full" for v in expand(dsa.get("sharing") or "full")]
-
-        kv_sharing = self.hparams.get("kv_sharing") or {}
-        if "source_layers_str" in kv_sharing:
-            self.layer_kv_source = [int(v) for v in kv_sharing["source_layers_str"].split(",")]
-        else:
-            assert not kv_sharing, f"unsupported kv_sharing config: {kv_sharing}"
-            self.layer_kv_source = list(range(n_layer))
-        assert len(self.layer_kv_source) == n_layer
-
-        # note: the rope dim of the indexer must match the main attention
-        assert dsa.get("rope_head_dim", -1) in (-1, self.hparams["qk_rope_head_dim"])
-        assert dsa.get("match_nope", False), "indexer must follow the NoPE pattern of the main attention"
-        assert dsa.get("ssmax_scaling", False), "indexer without SSMax is not supported"
-        assert self.hparams.get("q_lora_rank") is None
-        assert not self.hparams["yarn"].get("apply_scale", True), "only YaRN without magnitude scaling is supported"
-
-    def dequant_model(self):
-        quant_config = self.hparams.get("quantization")
-        if quant_config is not None:
-            assert quant_config["qformat_weight"] == "fp8_e4m3"
-            assert quant_config.get("qscheme_weight") == "BLOCK"
-            self.hparams["quantization_config"] = {
-                "quant_method": "fp8",
-                "weight_block_size": [quant_config["block_size_out"], quant_config["block_size_in"]],
-            }
-        return super().dequant_model()
-
-    def set_gguf_parameters(self):
-        # skip the MistralMoeModel overrides, they apply the deepseek2 YaRN scaling
-        super(MistralMoeModel, self).set_gguf_parameters()
-        MistralModel.set_mistral_config(self.gguf_writer, self.hparams)
-
-        dsa = self.hparams["dsa"]
-        self.gguf_writer.add_indexer_head_count(dsa["n_heads"])
-        self.gguf_writer.add_indexer_key_length(dsa["head_dim"])
-        self.gguf_writer.add_indexer_top_k(dsa["topk"])
-        self.gguf_writer.add_indexer_types(self.layer_indexer_full)
-        self.gguf_writer.add_layer_norm_eps(dsa.get("norm_eps", 1e-6))
-
-        self.gguf_writer.add_rope_pattern(self.layer_has_rope)
-        self.gguf_writer.add_kv_source_map(self.layer_kv_source)
-        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
-        self.gguf_writer.add_final_logit_softcapping(self.hparams["softmax_tanh_gating"])
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-
-        # exported separately with --dspark
-        if name.startswith("mtp_"):
-            return None
-
-        # experts need HF names for the merging logic in DeepseekV2Model
-        # note: bare "layers.N.experts" comes from the NVFP4 quantized_layers config
-        if ".experts." in name or name.endswith(".experts"):
-            name = "model." + name.replace(".experts", ".mlp.experts")
-            name = name.replace(".w1.", ".gate_proj.").replace(".w2.", ".down_proj.").replace(".w3.", ".up_proj.")
-
-        # skip the MistralMoeModel renaming, the qscale names are needed for block FP8 dequant
-        return super(MistralMoeModel, cls).filter_tensors((name, gen))
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "lerope.alpha":
-            # LeRoPE scales inv_freq by exp(alpha), ggml divides theta by the freq factors
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.exp(-data_torch.float()))
-            return
-
-        if bid is not None and name.endswith((".ssmax_scaling.s", ".ssmax_scaling.b")):
-            tensor = gguf.MODEL_TENSOR.INDEXER_SSMAX if ".indexer." in name else gguf.MODEL_TENSOR.ATTN_SSMAX
-            suffix = ".weight" if name.endswith(".s") else ".bias"
-            yield (self.format_tensor_name(tensor, bid, suffix), data_torch.float())
-            return
-
-        if bid is not None and ".indexer.q_input_norm." in name:
-            suffix = ".weight" if name.endswith(".weight") else ".bias"
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_NORM, bid, suffix), data_torch)
-            return
-
-        if bid is not None and ".indexer." in name:
-            name = "model." + name.replace(".attention.", ".self_attn.")
-
-        # NoPE layers have a larger non-rotary head, so wkv_b is split per layer
-        if bid is not None and name.endswith(".wkv_b.weight"):
-            n_head = self.hparams["num_attention_heads"]
-            v_head_dim = self.hparams["v_head_dim"]
-            qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
-            if not self.layer_has_rope[bid]:
-                qk_nope_head_dim += self.hparams["qk_rope_head_dim"]
-
-            assert data_torch.shape[0] == n_head * (qk_nope_head_dim + v_head_dim)
-
-            kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1])
-            k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
-            k_b = k_b.transpose(1, 2)
-
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid), k_b)
-            yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid), v_b)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-class MistralLarge4DSparkModel(MistralLarge4Model):
-    """Mistral Large 4 DSpark draft (mtp_dspark.*) as a DFLASH GGUF, export with --dspark"""
-    model_arch = gguf.MODEL_ARCH.DFLASH
-
-    _DSPARK_ROOT: dict[str, gguf.MODEL_TENSOR] = {
-        "backbone.target_projection.weight": gguf.MODEL_TENSOR.FC,
-        "backbone.target_norm.weight":       gguf.MODEL_TENSOR.ENC_OUTPUT_NORM,
-        "backbone.final_norm.weight":        gguf.MODEL_TENSOR.OUTPUT_NORM,
-        "backbone.mask_embedding":           gguf.MODEL_TENSOR.DSPARK_MASK_EMBD,
-        "autoregressive_head.embedding.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W1,
-        "autoregressive_head.output.weight":    gguf.MODEL_TENSOR.DSPARK_MARKOV_W2,
-        "autoregressive_head.joint_proj.weight": gguf.MODEL_TENSOR.DSPARK_JOINT_PROJ,
-        "autoregressive_head.joint_proj.bias":   gguf.MODEL_TENSOR.DSPARK_JOINT_PROJ,
-    }
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        mtp = self.hparams.get("mtp") or {}
-        dspark = mtp.get("dspark")
-        if dspark is None:
-            raise ValueError("Mistral Large 4 DSpark export needs the mtp.dspark config")
-        assert dspark.get("head_type") == "rnn", f"unsupported DSpark head type: {dspark.get('head_type')}"
-        assert dspark.get("draft_input") == "position_specific", f"unsupported DSpark draft input: {dspark.get('draft_input')}"
-
-        windows = {int(v) for v in (mtp.get("ragged_attention") or "").split(",") if v.strip()}
-        assert len(windows) == 1, f"DSpark draft layers must share one sliding window, got {mtp.get('ragged_attention')}"
-
-        self._n_stage = int(mtp["n_layers"])
-        self._depth = int(mtp["depth"])
-        self._window = windows.pop()
-        self._target_layers = [int(v) for v in mtp["target_activation_layers"].split(",")]
-
-        self.layer_has_rope = [True] * self._n_stage
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if not name.startswith("mtp_dspark."):
-            return None
-        name = name.removeprefix("mtp_dspark.")
-        if name.startswith("backbone.layers."):
-            return name.removeprefix("backbone."), gen
-        return f"dspark.{name}", gen
-
-    def set_vocab(self):
-        super().set_vocab()
-        # block uses mask_embd, mask token id is not used
-        self.gguf_writer.add_mask_token_id(0)
-
-    def set_gguf_parameters(self):
-        hp = self.hparams
-        w = self.gguf_writer
-        w.add_block_count(self._n_stage)
-        w.add_context_length(hp["max_position_embeddings"])
-        w.add_embedding_length(hp["hidden_size"])
-        w.add_feed_forward_length(hp["intermediate_size"])
-        w.add_head_count(hp["num_attention_heads"])
-        w.add_head_count_kv(1)
-        w.add_key_length(hp["kv_lora_rank"] + hp["qk_rope_head_dim"])
-        w.add_value_length(hp["kv_lora_rank"])
-        w.add_key_length_mla(hp["qk_nope_head_dim"] + hp["qk_rope_head_dim"])
-        w.add_value_length_mla(hp["v_head_dim"])
-        w.add_kv_lora_rank(hp["kv_lora_rank"])
-        w.add_rope_dimension_count(hp["qk_rope_head_dim"])
-        w.add_rope_freq_base(float(hp["rope_theta"]))
-        w.add_layer_norm_rms_eps(hp["rms_norm_eps"])
-        MistralModel.set_mistral_config(w, hp)
-        w.add_sliding_window(self._window)
-        w.add_final_logit_softcapping(hp["softmax_tanh_gating"])
-        w.add_block_size(self._depth)
-        # draft reads layer output = input of next layer
-        w.add_target_layers([layer + 1 for layer in self._target_layers])
-        w.add_sample_from_anchor(True)
-        w.add_has_confidence_head(False)
-        w.add_file_type(self.ftype)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name.startswith("dspark."):
-            key = name.removeprefix("dspark.")
-            if key not in self._DSPARK_ROOT:
-                raise ValueError(f"Unexpected DSpark tensor {name!r}")
-            suffix = ".bias" if key.endswith(".bias") else ".weight"
-            yield (self.format_tensor_name(self._DSPARK_ROOT[key], None, suffix), data_torch)
-            return
-
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/nemotron.py

@@ -428,19 +428,11 @@ class NemotronHModel(GraniteHybridModel):
     is_moe: bool = False
     supports_mtp_export = True
     _experts: list[dict[str, Tensor]] | None = None
-    # tokens past the embedding rows (left out of the vocab), and special tokens that are printed as text
-    _vocab_skip: tuple[str, ...] = ()
-    _user_defined_tokens: re.Pattern[str] | None = None
 
     _SSM_LAYER_TYPES = {"mamba", "linear_attention"}
     _ATTN_LAYER_TYPES = {"attention", "full_attention"}
     _MLP_LAYER_TYPES = {"moe"}
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        if ".mixer.experts." in name:
-            name = name.replace("backbone.", "model.").replace(".mixer.experts.", ".mlp.experts.")
-        return super()._map_fp8_weight_names(name)
-
     def __init__(self, *args, **kwargs):
         # We have to determine the correct model architecture (MoE vs non-MoE) before
         # calling the parent __init__. This is because the parent constructor
@@ -619,11 +611,7 @@ class NemotronHModel(GraniteHybridModel):
         vocab_size = -(vocab_size // -pad_vocab) * pad_vocab
         self.hparams["vocab_size"] = vocab_size
 
-        # draft-model converters (DFlash, LiLiCorr) call this with their own instance, which has neither hook
-        vocab_skip = getattr(self, "_vocab_skip", ())
-        user_defined_tokens = getattr(self, "_user_defined_tokens", None)
-
-        assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size  # ty: ignore[unresolved-attribute]
+        assert max(tokenizer.vocab.values()) < vocab_size  # ty: ignore[unresolved-attribute]
 
         tokpre = self.get_vocab_base_pre(tokenizer)
 
@@ -645,8 +633,7 @@ class NemotronHModel(GraniteHybridModel):
                         if previous_token != token:
                             logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
 
-                    printed = user_defined_tokens is not None and user_defined_tokens.fullmatch(token)
-                    if (added_tokens_decoder[i].special or self.does_token_look_special(token)) and not printed:
+                    if added_tokens_decoder[i].special or self.does_token_look_special(token):
                         toktypes.append(gguf.TokenType.CONTROL)
                     else:
                         token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ")  # pre-normalize user-defined spaces
@@ -842,140 +829,3 @@ class NemotronHPuzzleModel(NemotronHModel):
         if item[0].startswith("mtp."):
             return None
         return super().filter_tensors(item)
-
-
-# NeMo speechlm2 SALM checkpoint: a Nemotron-H decoder under llm.*, the speech encoder under perception.*
-@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration")
-@ModelBase.example("nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B")
-class NemotronTranscribeModel(NemotronHModel):
-    model_arch = gguf.MODEL_ARCH.NEMOTRON_H
-    # the audio placeholder (id vocab_size) has no embedding row; speaker, event and style tags print with the transcript
-    _vocab_skip = ("<|audio|>",)
-    _user_defined_tokens = re.compile(r"<spk:\d+>|<ev:[a-z_]+>|<sty:(?:stt|end):[a-z_]+>")
-    _renames = {
-        "llm.model.embed_tokens.weight": "backbone.embeddings.weight",
-        "llm.model.norm.weight":         "backbone.norm_f.weight",
-        "llm.lm_head.weight":            "lm_head.weight",
-    }
-    # grouped experts: [n_expert, n_in, n_out] -> [n_expert, n_out, n_in]
-    _grouped_experts = {"gate_and_up_projs": "up_proj", "down_projs": "down_proj"}
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if name.startswith(("perception.", "ctc_timestamp.")) or "_extra_state" in name:
-            return None
-        if not name.startswith("llm."):
-            # the FP8 and NVFP4 releases store the decoder as backbone.*
-            raise ValueError(f"unexpected tensor {name}: only the BF16 checkpoint layout (llm.*) is supported")
-        if name in cls._renames:
-            name = cls._renames[name]
-        elif name.startswith("llm.model.layers."):
-            name = "backbone.layers." + name[len("llm.model.layers."):]
-        elif name.startswith("llm.mtp.layers."):
-            name = "mtp.layers." + name[len("llm.mtp.layers."):]
-        else:
-            raise ValueError(f"unexpected decoder tensor {name}")
-        return super().filter_tensors((name.replace("._fp32_params.", "."), gen))
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        leaf = name.rsplit(".", 1)[-1]
-        if ".mixer.experts." in name and leaf in self._grouped_experts:
-            if name.startswith("mtp."):
-                assert self._mtp_bid is not None, f"{name}: MTP head is not exported"
-                bid = self._mtp_bid
-            assert bid is not None
-            data_torch = data_torch.permute(0, 2, 1).contiguous()
-            name = f"model.layers.{bid}.mlp.experts.{self._grouped_experts[leaf]}.weight"
-            yield from ModelBase.modify_tensors(self, data_torch, name, bid)
-            return
-        yield from super().modify_tensors(data_torch, name, bid)
-
-
-@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration")
-class NemotronTranscribeAudioModel(MmprojModel):
-    # NeMo ParallelExpertEncoder: ASR and speaker transformers, speaker activity fused into the ASR states
-    has_vision_encoder = False
-    has_audio_encoder = True
-
-    def get_audio_config(self) -> dict[str, Any] | None:
-        asr = self.global_config["pe_encoder_config"]["asr_encoder_cfg"]
-        return {
-            "hidden_size":         asr["d_model"],
-            "intermediate_size":   int(float(asr.get("ff_expansion", 4)) * asr["d_model"]),
-            "num_hidden_layers":   asr["n_layers"],
-            "num_attention_heads": asr["n_heads"],
-        }
-
-    @staticmethod
-    def _rope_dims(cfg: dict[str, Any]) -> int:
-        assert cfg.get("self_attention_model") == "rope" and cfg.get("subsampling", "feature_stacking") == "feature_stacking"
-        assert cfg.get("attn_mode", "full") == "full" and not cfg.get("xscaling", False) and cfg.get("pre_block_norm", True)
-        assert float(cfg.get("rope_base", 10000.0)) == 10000.0
-        n_rot = int(cfg["d_model"] // cfg["n_heads"] * float(cfg.get("rotary_fraction", 1.0)))
-        assert n_rot >= 2 and n_rot % 2 == 0
-        return n_rot
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        pe = self.global_config["pe_encoder_config"]
-        asr = pe["asr_encoder_cfg"]
-        diar = pe["diarization_model_cfg"]
-        spk = diar["encoder"]
-        assert pe.get("asr_encoder_type") == "transformer" and pe.get("speaker_feature_mode", "thresholded") == "thresholded"
-        assert pe.get("asr_normalize_type") == "per_feature" and pe.get("diar_normalize_type") in (None, "NA")
-        assert asr["subsampling_factor"] == spk["subsampling_factor"] == 8 and asr["feat_in"] == spk["feat_in"]
-        assert self._rope_dims(spk) == spk["d_model"] // spk["n_heads"] and not spk.get("qkv_bias", False)
-        assert diar.get("high_resolution", False)
-        # constants in clip: the ParallelExpertEncoder online settings and the Sortformer streaming config
-        assert (pe.get("online_inference_length", 500), pe.get("chunk_left_context", 50), pe.get("chunk_right_context", 50),
-                pe.get("diar_spkcache_len", 188), pe.get("diar_fifo_len", 40), pe.get("diar_spkcache_update_period", 300),
-                pe.get("speaker_activity_threshold", 0.5)) == (500, 50, 50, 188, 40, 300, 0.5), \
-            "streaming settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp"
-        sm = diar["sortformer_modules"]
-        assert (sm["spkcache_sil_frames_per_spk"], sm["pred_score_threshold"], sm["scores_boost_latest"], sm["strong_boost_rate"],
-                sm["weak_boost_rate"], sm["min_pos_scores_rate"], sm["use_learnable_sil_emb"]) == (1, 0.25, 0.05, 0.75, 1.5, 0.5, True), \
-            "speaker cache settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp"
-
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_TRANSCRIBE)
-        self.gguf_writer.add_audio_num_mel_bins(asr["feat_in"])
-        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-        self.gguf_writer.add_audio_subsampling_factor(asr["subsampling_factor"])
-        self.gguf_writer.add_audio_rope_dims(self._rope_dims(asr))
-        self.gguf_writer.add_audio_spk_block_count(spk["n_layers"])
-        self.gguf_writer.add_audio_spk_head_count(spk["n_heads"])
-
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # ParallelExpertEncoder._build_sinusoid_position_encoding
-        pe = self.global_config["pe_encoder_config"]
-        n_spk = int(pe["diarization_model_cfg"]["sortformer_modules"]["num_spks"])
-        d = int(pe["asr_encoder_cfg"]["d_model"])
-        scale = float(pe.get("spk_kernel_scale", 1.0))
-        pos = torch.arange(n_spk, dtype=torch.float32)[:, None]
-        div = torch.exp(torch.arange(0, d, 2, dtype=torch.float32) * -(np.log(10000.0) / d))
-        kernel = torch.zeros(n_spk, d, dtype=torch.float32)
-        kernel[:, 0::2] = torch.sin(pos * div)
-        kernel[:, 1::2] = torch.cos(pos * div)
-        # stored as [d, n_spk]: the graph multiplies it with the speaker activity directly
-        yield self.format_tensor_name(gguf.MODEL_TENSOR.A_FUSION_SPK_KERNEL, suffix=""), (kernel * scale).T.contiguous()
-
-    def tensor_force_quant(self, name, new_name, bid, n_dims):
-        if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE, None):
-            return gguf.GGMLQuantizationType.F16  # ggml_conv_1d kernel
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-        if not name.startswith("perception.") or "_extra_state" in name:
-            return None
-        # training-only head; the speaker model's unused front end
-        if name.startswith(("perception.encoder.diarization_model.sortformer_modules.hidden_to_spks.",
-                            "perception.encoder.diarization_model.preprocessor.")):
-            return None
-        return name, gen
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "perception.preprocessor.featurizer.fb":
-            data_torch = data_torch.squeeze(0)
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/pockettts.py

@@ -5,7 +5,6 @@ from pathlib import Path
 from typing import Any, Iterable, TYPE_CHECKING
 
 import torch
-import torch.nn.functional as F
 
 if TYPE_CHECKING:
     from torch import Tensor
@@ -46,102 +45,6 @@ def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]:
         return {name: tuple(part[name].shape) for name in part.keys()}
 
 
-def _has_voice_encoder(dir_model: Path) -> bool:
-    # without voice cloning, the mimi encoder weights are all zero
-    part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors")
-    with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part:
-        nonzero = {bool(part[name].mmap_bytes().any()) for name in part.keys() if name.startswith("mimi.encoder")}
-    if len(nonzero) != 1:
-        raise ValueError(f"{dir_model}: the mimi encoder is partly zeroed out")
-    return nonzero.pop()
-
-
-def _load_safetensors(path: Path, prefixes: tuple[str, ...]) -> dict[str, Tensor]:
-    dtypes = {"F32": torch.float32, "BF16": torch.bfloat16, "I64": torch.int64}
-    with gguf.utility.SafetensorsLocal(path) as part:
-        return {name: torch.from_numpy(part[name].mmap_bytes()).view(dtypes[part[name].dtype]).reshape(part[name].shape)
-                for name in part.keys() if name.startswith(prefixes)}
-
-
-def _rope_pairs(x: Tensor, n_head: int, rope_theta: float) -> Tensor:
-    # ggml NORM mode rope, x: [n_pos, n_embd]
-    n_pos, n_embd = x.shape
-    head_dim = n_embd // n_head
-    angle = torch.arange(n_pos, dtype=x.dtype)[:, None, None] * \
-        torch.pow(rope_theta, -torch.arange(0, head_dim, 2, dtype=x.dtype) / head_dim)
-    x = x.view(n_pos, n_head, head_dim // 2, 2)
-    return torch.stack([x[..., 0] * angle.cos() - x[..., 1] * angle.sin(),
-                        x[..., 0] * angle.sin() + x[..., 1] * angle.cos()], -1).reshape(n_pos, n_embd)
-
-
-def _voice_prompt_embd(weights: dict[str, Tensor], kv: list[Tensor], n_head: int, rope_theta: float) -> tuple[Tensor, float]:
-    # the voice state only has K/V of each layer (kv[il] is [k/v, pos, n_embd]): layer 0 V gives each frame up to its scale, layer 1 V gives the scale
-    # returns the frames without bos_before_voice, and the max relative K/V error when the frames are prompted again
-    w = {name: t.double() for name, t in weights.items()}
-    kv = [t.double() for t in kv]
-    k0, v0, v1 = kv[0][0], kv[0][1], kv[1][1]
-    n_pos, n_embd = v0.shape
-    head_dim = n_embd // n_head
-
-    def p(il: int, name: str) -> Tensor:
-        return w[f"flow_lm.transformer.layers.{il}.{name}"]
-
-    def norm(x: Tensor, il: int, name: str) -> Tensor:
-        return F.layer_norm(x, (n_embd,), p(il, f"{name}.weight"), p(il, f"{name}.bias"), 1e-5)
-
-    def heads(t: Tensor) -> Tensor:
-        return t.reshape(n_pos, n_head, head_dim).transpose(0, 1)
-
-    def rows(t: Tensor) -> Tensor:
-        return t.transpose(0, 1).reshape(n_pos, n_embd)
-
-    def attn(il: int, h: Tensor, k: Tensor, v: Tensor) -> Tensor:
-        q = _rope_pairs(h @ p(il, "self_attn.in_proj.weight").chunk(3)[0].T, n_head, rope_theta)
-        return rows(F.scaled_dot_product_attention(heads(q), heads(k), heads(v), is_causal=True)) @ p(il, "self_attn.out_proj.weight").T
-
-    def ffn(il: int, x: Tensor) -> Tensor:
-        return F.gelu(norm(x, il, "norm2") @ p(il, "linear1.weight").T) @ p(il, "linear2.weight").T
-
-    wv = p(0, "self_attn.in_proj.weight").chunk(3)[2]
-    h = torch.linalg.solve(wv, v0.T).T                   # norm1(x)
-    z = (h - p(0, "norm1.bias")) / p(0, "norm1.weight")  # (x - mean(x)) / std(x)
-
-    # fit z on [speaker_proj, 1] to get the latent direction back
-    proj = w["flow_lm.speaker_proj_weight"]
-    fit = torch.linalg.lstsq(torch.cat([proj, torch.ones(n_embd, 1, dtype=proj.dtype)], 1), z[1:].T).solution
-    frames = (proj @ fit[:-1]).T
-
-    a0 = attn(0, h, k0, v0)[1:]
-    wv1 = p(1, "self_attn.in_proj.weight").chunk(3)[2]
-
-    def err(scale: Tensor) -> Tensor:  # per frame relative error of layer 1 V
-        x = scale[:, None] * frames + a0
-        x = x + ffn(0, x)
-        return (norm(x, 1, "norm1") @ wv1.T - v1[1:]).norm(dim=1) / v1[1:].norm(dim=1)
-
-    grid = torch.logspace(-3, 3, 49, dtype=torch.float64)
-    best = torch.stack([err(s.expand(n_pos - 1)) for s in grid]).argmin(0)
-    lo, hi = grid[(best - 1).clamp(min=0)], grid[(best + 1).clamp(max=len(grid) - 1)]
-    for _ in range(40):  # golden section search, all frames at once
-        m1, m2 = hi - 0.618 * (hi - lo), lo + 0.618 * (hi - lo)
-        left = err(m1) < err(m2)
-        lo, hi = torch.where(left, lo, m1), torch.where(left, m2, hi)
-    frames = (lo + hi)[:, None] / 2 * frames
-
-    # prompt the frames again and compare K/V with the shipped state
-    x = torch.cat([w["flow_lm.bos_before_voice"].reshape(1, n_embd), frames])
-    drift = 0.0
-    for il, (k_ref, v_ref) in enumerate(kv):
-        h = norm(x, il, "norm1")
-        _, wk, wv = p(il, "self_attn.in_proj.weight").chunk(3)
-        k, v = _rope_pairs(h @ wk.T, n_head, rope_theta), h @ wv.T
-        for a, b in ((k, k_ref), (v, v_ref)):
-            drift = max(drift, ((a - b).norm(dim=1) / b.norm(dim=1)).median().item())
-        x = x + attn(il, h, k, v)
-        x = x + ffn(il, x)
-    return frames, drift
-
-
 @ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model))
 def _load_hparams(dir_model: Path) -> dict[str, Any]:
     logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes")
@@ -291,27 +194,22 @@ class PocketTTSMmprojModel(MmprojModel):
         (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V),
     )
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.has_audio_encoder = _has_voice_encoder(self.dir_model)
-
     def set_gguf_parameters(self):
         self.gguf_writer.add_file_type(self.ftype)
         assert self.hparams_audio is not None
 
         # voice-prompt encoder: mimi encoder + speaker_proj
-        if self.has_audio_encoder:
-            self.gguf_writer.add_clip_has_audio_encoder(True)
-            # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
-            self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
-            self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
-            self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
-            self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
-            self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
-            self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
-            self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
-            # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
-            self.gguf_writer.add_audio_num_mel_bins(1)
+        self.gguf_writer.add_clip_has_audio_encoder(True)
+        # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models
+        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC)
+        self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
+        self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
+        self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
+        self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
+        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+        # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
+        self.gguf_writer.add_audio_num_mel_bins(1)
 
         # generation: flow-matching decoder + mimi decoder
         # the SEANet and flow net hparams are constant across the family, clip.cpp holds them
@@ -331,57 +229,12 @@ class PocketTTSMmprojModel(MmprojModel):
         # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
         if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"):
             return gguf.GGMLQuantizationType.F16
-        # voices are read on the host
-        if new_name.startswith("voice."):
-            return gguf.GGMLQuantizationType.F32
         return False
 
-    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
-        # embeddings/<name>.safetensors hold the K/V state after the voice prompt (see get_state_for_audio_prompt() in the reference), turn it back into prompt embeddings
-        paths = sorted((self.dir_model / "embeddings").glob("*.safetensors"))
-        if not paths:
-            return
-        weights = _load_safetensors(self.dir_model / "model.safetensors",
-                                    ("flow_lm.transformer.layers.", "flow_lm.speaker_proj_weight", "flow_lm.bos_before_voice"))
-        n_layer = self.global_config["num_hidden_layers"]
-        for path in paths:
-            state = _load_safetensors(path, ("transformer.layers.",))
-            kv = []
-            for il in range(n_layer):
-                prefix = f"transformer.layers.{il}.self_attn/"
-                if any(prefix + k not in state for k in ("cache", "offset")):
-                    break
-                n_pos = int(state[prefix + "offset"].item())
-                # older packs have no pad entry
-                if prefix + "pad" in state and int(state[prefix + "pad"].item()) != 0:
-                    break
-                kv.append(state[prefix + "cache"][:, 0, :n_pos].flatten(2))  # [k/v, pos, n_embd]
-            if len(kv) != n_layer or any(t.shape != kv[0].shape for t in kv) or kv[0].shape[1] < 2 or \
-                    kv[0].shape[2] != self.global_config["hidden_size"]:
-                logger.warning(f"voice {path.stem}: skipped, not an unpadded voice state of this backbone")
-                continue
-            embd, drift = _voice_prompt_embd(weights, kv, self.global_config["num_attention_heads"],
-                                             self.global_config["rope_theta"])
-            # some packs ship a state made with other weights, the reference still uses it: keep up to 50% error, warn above 5%
-            if drift > 0.5:
-                logger.warning(f"voice {path.stem}: skipped, its state does not belong to these weights (error {drift:.1e})")
-                continue
-            log = logger.warning if drift > 0.05 else logger.info
-            log(f"voice {path.stem}: {embd.shape[0]} frames, prompting them again gives the shipped state within {drift:.1e}")
-            yield (f"voice.{path.stem}", embd.float())
-
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         del bid  # the block index of the mimi transformers is parsed here, not by the base class
         T = gguf.MODEL_TENSOR
 
-        if name.startswith("voice."):
-            yield (name, data_torch)
-            return
-
-        if not self.has_audio_encoder and (name.startswith(("mimi.encoder", "mimi.downsample."))
-                                           or name == "flow_lm.speaker_proj_weight"):
-            return  # voice-prompt encoder
-
         if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"):
             return  # folded into the backbone embedding table
         if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."):

conversion/qwen.py

@@ -83,15 +83,6 @@ class Qwen2Model(TextModel):
 class Qwen2MoeModel(TextModel):
     model_arch = gguf.MODEL_ARCH.QWEN2MOE
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        if name.removesuffix(".weight").endswith(".mlp.experts.gate_up_proj"):
-            bid = next(int(part) for part in self.map_tensor_name(name).split(".") if part.isdecimal())
-            return (
-                self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid),
-                self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid),
-            )
-        return super()._map_fp8_weight_names(name)
-
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
         if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
@@ -391,16 +382,6 @@ class _QwenMtpMixin:
 class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
     model_arch = gguf.MODEL_ARCH.QWEN3NEXT
 
-    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        if name.endswith(".linear_attn.in_proj_qkvz.weight"):
-            bid = next(int(part) for part in self.map_tensor_name(name).split(".") if part.isdecimal())
-            # Both split projections keep the original per-tensor scales.
-            return (
-                self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid),
-                self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_GATE, bid),
-            )
-        return super()._map_fp8_weight_names(name)
-
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
         self.gguf_writer.add_ssm_conv_kernel(self.hparams["linear_conv_kernel_dim"])

convert_hf_to_gguf.py

@@ -58,7 +58,7 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--outtype", type=str, choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"], default="auto",
-        help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto to preserve supported FP8 weights and otherwise select the highest-fidelity 16-bit float type",
+        help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type",
     )
     parser.add_argument(
         "--bigendian", action="store_true",
@@ -127,7 +127,7 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--dspark", action="store_true",
-        help="Export only the DeepSeek-V4 or Mistral Large 4 DSpark draft tensors as a separate GGUF.",
+        help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",
     )
     parser.add_argument(
         "--mistral-format", action="store_true",
@@ -255,9 +255,6 @@ def main() -> None:
             assert hparams.get("vision_encoder") is not None, "This model does not support multimodal"
             from conversion.pixtral import PixtralModel
             model_class = PixtralModel
-        elif hparams.get("moe") is not None and hparams.get("dsa") is not None:
-            from conversion.mistral4 import MistralLarge4Model
-            model_class = MistralLarge4Model
         elif hparams.get("moe") is not None:
             from conversion.mistral import MistralMoeModel
             model_class = MistralMoeModel
@@ -270,16 +267,10 @@ def main() -> None:
             sys.exit(1)
 
         if args.dspark:
-            if is_mistral_format or model_architecture == "MistralLarge4ForConditionalGeneration":
-                if model_class.__name__ != "MistralLarge4Model":
-                    logger.error("--dspark is only supported for Mistral Large 4 among Mistral-format models")
-                    sys.exit(1)
-                from conversion.mistral4 import MistralLarge4DSparkModel
-                model_class = MistralLarge4DSparkModel
-            elif model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"):
-                logger.error("--dspark is only supported for DeepseekV4ForCausalLM, DeepseekV41ForCausalLM and Mistral Large 4")
+            if is_mistral_format or model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"):
+                logger.error("--dspark is only supported for DeepseekV4ForCausalLM and DeepseekV41ForCausalLM")
                 sys.exit(1)
-            elif model_architecture == "DeepseekV41ForCausalLM":
+            if model_architecture == "DeepseekV41ForCausalLM":
                 from conversion.deepseek41 import DeepseekV41DSparkModel
                 model_class = DeepseekV41DSparkModel
             else:

convert_hf_to_gguf_update.py

@@ -198,7 +198,6 @@ pre_computed_hashes = [
     # jina-v2-de variants
     {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
     {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
-    {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"},
     # lfm2 variants
     {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
     # hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges

docs/backend/CANN.md

@@ -161,7 +161,7 @@ lspci -n | grep -Eo '19e5:d[0-9a-f]{3}' | cut -d: -f2
 | FP16                   | Support | Support |
 | Q8_0                   | Support | Partial |
 | Q4_0                   | Support | Partial |
-| BF16                   | Support |    No   |
+| BF16                   | Support |         |
 
 > **310P note**
 > - `Q8_0`: data transform / buffer path is implemented, and `GET_ROWS` is supported, but quantized `MUL_MAT` / `MUL_MAT_ID` are not supported.
@@ -274,9 +274,7 @@ cmake --build build --config release
 
     **Notes**:
 
-      - On 910B the CANN backend supports FP16, BF16 and Q4_0/Q8_0 models. On 310P, BF16 is not
-        supported and quantized `MUL_MAT` / `MUL_MAT_ID` are not implemented, so only FP16 models
-        are supported (see [DataType Supports](#datatype-supports)).
+      - CANN backend only supports FP16/Q4_0/Q8_0 models currently.
 
 2. **Launch inference**
 
@@ -357,7 +355,3 @@ Enable ACL graph execution during the prefill stage, default is false. This opti
 ### GGML_CANN_OPERATOR_FUSION
 
 Enable operator fusion during computation, default is false. This option fuses compatible operators (e.g., ADD + RMS_NORM) to reduce overhead and improve performance.
-
-### GGML_CANN_NO_PINNED
-
-Disable pinned (host-locked) host buffers. When it is set (any value), `aclrtMallocHost` is not called: the CANN host buffer type allocates regular CPU memory instead and the backend reports no `host_buffer` capability, so host-side staging uses pageable memory. Use it when pinned host allocation fails, or when the pinned memory is needed by other processes on the host.

docs/backend/SYCL.md

@@ -784,7 +784,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_TARGET   | INTEL *(default)*                     | Set the SYCL target device type.            |
 | GGML_SYCL_DEVICE_ARCH | Optional                           | Set the SYCL device architecture. Setting the device architecture can improve the performance. See the table [--offload-arch](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/OffloadDesign.md#--offload-arch) for a list of valid architectures. |
 | GGML_SYCL_F16      | OFF *(default)* \|ON *(optional)*     | Enable FP16 build with SYCL code path. [1.](#sycl-f16-footnote) |
-| GGML_SYCL_SUPPORT_XMX | ON *(default)* \|OFF *(Optional)* | Build the code paths that use XMX (DPAS) instructions. Set OFF for AOT builds targeting iGPUs without XMX, which otherwise fail to compile. |
 | GGML_SYCL_GRAPH    | ON *(default)* \|OFF *(Optional)*     | Enable build with [SYCL Graph extension](https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc). |
 | GGML_SYCL_DNN      | ON *(default)* \|OFF *(Optional)*     | Enable build with oneDNN.                   |
 | GGML_SYCL_HOST_MEM_FALLBACK | ON *(default)* \|OFF *(Optional)* | Allow host memory fallback when device memory is full during quantized weight reorder. Enables inference to continue at reduced speed (reading over PCIe) instead of failing. Requires Linux kernel 6.8+. |
@@ -813,7 +812,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
 | GGML_SYCL_ENABLE_DNN | 0 or 1 (default)| Enable running computations through oneDNN and always use oneMKL. |
 | GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. |
 | GGML_SYCL_FA_ONEDNN_MAX_KV | 0 (default, disabled) or positive integer | By default (0), all sequences are handled by the oneDNN fused SDPA path, regardless of KV length; a positive value caps that length, past which sequences fall back to the native kernel. If GPU driver watchdog resets (DEVICE_LOST) occur during long-context inference, set this near the context depth where they start, e.g. 24576. |
-| GGML_SYCL_FA_DPAS | 1 (default) or 0 | Enable the XMX flash-attention kernel for a q4_0 or q8_0 KV cache (`-ctk q4_0 -ctv q4_0` or `-ctk q8_0 -ctv q8_0`) on Xe2 or newer GPUs (Arc B-series, Lunar Lake, Panther Lake, Wildcat Lake). It reads the quantized cache directly for decode and batches of up to 31 tokens, head sizes 64, 96, 128, 256 and 512, up to 32 query heads per KV head. Set to 0 to use the native SYCL flash-attention kernels. |
 | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. |
 | GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` |
 | GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. |
@@ -848,18 +846,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
    Then benchmark with your preferred tool, such as `llama-bench`.
    For graph activity details (including reuse), set `GGML_SYCL_DEBUG=1` and look for `[SYCL-GRAPH]` messages.
 
-### Flash-attention kernels
-
-Attention dispatches on shape and hardware.
-
-DPAS is the quantized path. q4_0 or q8_0 KV cache, decode and batches up to 31 tokens, Xe2 and newer only. It issues dpas straight on the quantized blocks and never materialises an f16 copy. vISA gets used here because joint_matrix has no s4 type.
-
-oneDNN does XMX through SDPA but converts KV to f16 first, so it ends up on prefill: 32+ tokens and 1K+ keys. MKL is the prompt processing equivalent through oneMKL GEMM, also f16.
-
-VEC/TILE are plain kernels with no XMX in them. They run when DPAS is off or the hardware doesn't support it.
-
-joint_matrix belongs to the matmul GEMMs. Attention doesn't use it.
-
 ## Compile-time Flags
 
 Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spot.

docs/build-s390x.md

@@ -249,7 +249,7 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
 | MXFP4      | ✅           | ❓    | ❓     |
 | Q5_0       | ✅           | ❓    | ❓     |
 | Q5_1       | ✅           | ❓    | ❓     |
-| Q8_0       | ✅           | ✅    | ❓     |
+| Q8_0       | ✅           | ❓    | ❓     |
 | Q2_K       | 🚫           | ❓    | ❓     |
 | Q3_K       | ✅           | ❓    | ❓     |
 | Q4_K       | ✅           | ❓    | ❓     |
@@ -266,11 +266,11 @@ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongl
 | IQ1_M      | 🚫           | ❓    | ❓     |
 | IQ4_NL     | ✅           | ❓    | ❓     |
 | IQ4_XS     | ✅           | ❓    | ❓     |
-| FP32->FP16 | ✅           | ❓    | ❓     |
+| FP32->FP16 | 🚫           | ❓    | ❓     |
 | FP16->FP32 | 🚫           | ❓    | ❓     |
 
 -   ✅ - acceleration available
 -   🚫 - acceleration unavailable, will still run using scalar implementation
 -   ❓ - acceleration unknown, please contribute if you can test it yourself
 
-Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Nov 3, 2026.
+Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Sep 8, 2026.

docs/build.md

@@ -481,7 +481,7 @@ cmake --build build --config Release
 Now you can load the model in conversation mode using `Vulkan`
 
 ```sh
-build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2
+build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 -cnv
 ```
 
 **MSYS2**

docs/ops.md

@@ -14,22 +14,22 @@ Legend:
 
 | Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
 |-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|
-|                              ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                              ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 |                           ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                           ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                            CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                           CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                             CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
-|                       CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
+|                       CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ |
 |                          CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
-|                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                       CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
@@ -38,49 +38,49 @@ Legend:
 |                              CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |               CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |          CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                           CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                     DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                     DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                      DSV4_HC_PRE | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                              DUP | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                              ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                   FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
-|                            FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                  GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                      GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                      GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                       GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
 |                    GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
 |                       GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                      HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                        HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                           IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
+|                        IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+|                          L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                       LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                              LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ |
-|                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
-|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
+|                 MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 |
 |                       MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
-|                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                     MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                     MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
-|                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                  MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                  MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                     MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                     MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                              NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                   OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                     OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
@@ -89,46 +89,45 @@ Legend:
 |                   PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                          POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                          POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                           REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
 |                      REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                         RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                             ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                             ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 |                        RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                            SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
 |                         SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
-|                              SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                        SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
 |                              SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                    SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
-|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
+|                        SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
 |                              SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                             SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                         SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
-|                             STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                              SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                         SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
+|                             STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                              SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                              SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
 |                         SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
-|                           SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                     SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
+|                           SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                     SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
 |                       SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                             TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                             TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |               TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-|                            TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
-|                       TOP_K_MASK | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-|                              TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-|                            TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
+|                              TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
+|                            TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                          UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
 |                            XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |

docs/ops/WebGPU.csv

@@ -12326,16 +12326,8 @@
 "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
 "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9","support","0","no","WebGPU"
 "WebGPU: WebGPU","ROLL","shift0=3,shift1=-2,shift3=1,shift4=-1","support","0","no","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=3.000000,stop=3.500000,step=1.000000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-10.000000,stop=10.000000,step=0.300000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=62.750000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.000000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.250000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=254.750000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.000000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.250000,step=0.250000","support","1","yes","WebGPU"
-"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","1","yes","WebGPU"
+"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","0","no","WebGPU"
+"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","0","no","WebGPU"
 "WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","0","no","WebGPU"
 "WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU"
 "WebGPU: WebGPU","CUMSUM","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU"

docs/speculative.md

@@ -100,9 +100,8 @@ llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DSpark.gguf \
 
 `--spec-draft-n-max` is clamped to the draft model's trained block size.
 
-`--spec-draft-p-min P` truncates each drafted block at the first position whose predicted
-acceptance (from the draft's confidence head) falls below `P` (default 0 = disabled).
-Values above 0 require a draft with a confidence head.
+`--spec-draft-conf-min P` truncates each drafted block at the first position whose predicted
+acceptance (from the draft's confidence head, if present) falls below `P` (default 0 = disabled).
 
 Currently only drafts with a Qwen3 backbone are supported; support for other backbones
 (e.g. Gemma4) is planned.

examples/realtime_client.py

@@ -1,211 +0,0 @@
-#!/usr/bin/env python3
-"""Example client for the llama-server /v1/realtime websocket, with the openai SDK.
-
-Start a server with an audio model, e.g. LFM3-Audio:
-
-    llama-server -hf LiquidAI/LFM3-Audio-2B-GGUF
-
-Spoken conversation, the audio of each wav file is one user turn. It is sent in chunks as it would come from a
-microphone, the server prefills it as it arrives so the response starts right after the last chunk:
-
-    python examples/realtime_client.py --instructions "Respond with interleaved text and audio." question.wav
-
-Output modalities: left out, the model picks; "text", "audio" or "text,audio" (interleaved) force the response.
-Without --instructions, a forced mode comes with the system prompt of the LFM3-Audio or LFM3-Omni model card:
-
-    python examples/realtime_client.py --output-modalities text,audio --out answer.wav question.wav
-
-Tool calls: --tools gives the conversation a demo tool, the client runs its calls and sends the results back:
-
-    python examples/realtime_client.py --tools --text "Set the volume to 30."
-
-Text turns go in with --text, transcription sessions transcribe each file:
-
-    python examples/realtime_client.py --text "What is the capital of France?"
-    python examples/realtime_client.py --type transcription --language en speech.wav
-
-Live transcription models (Voxtral Realtime, LFM3-Audio live ASR) transcribe while the audio is sent, a transcription
-session opens with ?intent=transcription:
-
-    python examples/realtime_client.py --type transcription speech.wav
-
-Requirements: pip install "openai[realtime]" numpy
-"""
-
-import argparse
-import asyncio
-import base64
-import json
-import time
-import wave
-
-import numpy as np
-from openai import AsyncOpenAI
-
-
-# system prompts of the LiquidAI/LFM3-Audio-2B and LiquidAI/LFM3-Omni-2B model cards for a forced output mode
-MODE_INSTRUCTIONS = {
-    "audio": {"text,audio": "Respond with interleaved text and audio."},
-    "omni":  {"text": "Respond in text.", "text,audio": "Respond in interleaved text and audio."},
-}
-# and with tools: calls, or an answer in interleaved text and speech
-TOOLS_INSTRUCTIONS = "Respond with function calls, and interleaved text and audio.\n\nList of tools:"
-
-# the demo tool of --tools
-DEMO_TOOLS = [{
-    "type": "function",
-    "name": "set_volume",
-    "description": "Set the device speaker volume.",
-    "parameters": {
-        "type": "object",
-        "properties": {"volume": {"type": "integer", "description": "Volume from 0 to 100."}},
-        "required": ["volume"],
-    },
-}]
-
-
-def run_tool(name, arguments):
-    if name == "set_volume":
-        return {"volume": json.loads(arguments)["volume"]}
-    return {"error": f"unknown tool {name}"}
-
-
-def read_wav(path, rate):
-    # PCM16 wav -> mono int16 at the given rate
-    with wave.open(path, "rb") as f:
-        assert f.getsampwidth() == 2, "the example reads PCM16 wav files"
-        x = np.frombuffer(f.readframes(f.getnframes()), dtype=np.int16).astype(np.float32)
-        x = x.reshape(-1, f.getnchannels()).mean(axis=1)
-        sr = f.getframerate()
-    if sr != rate:
-        n = int(round(len(x) * rate / sr))
-        x = np.interp(np.arange(n) * sr / rate, np.arange(len(x)), x)
-    return np.clip(x, -32768, 32767).astype(np.int16)
-
-
-def write_wav(path, pcm, rate):
-    with wave.open(path, "wb") as f:
-        f.setnchannels(1)
-        f.setsampwidth(2)
-        f.setframerate(rate)
-        f.writeframes(pcm.tobytes())
-
-
-async def main(args):
-    # the SDK derives a wss:// URL from base_url, a plain http server needs ws://
-    client = AsyncOpenAI(base_url=args.url, api_key=args.api_key, websocket_base_url=args.url.replace("http", "ws", 1))
-    query = {"intent": "transcription"} if args.type == "transcription" else {}
-    async with client.realtime.connect(model=args.model, extra_query=query) as conn:
-        ev = await conn.recv()
-        if ev.type == "error":
-            print("error:", ev.error.message)
-            return
-        # the audio goes in at the rate of the model, PCM16 mono
-        session_info = ev.model_dump(warnings=False)["session"]
-        audio_fmt = session_info["audio"]
-        card = "omni" if "omni" in str(session_info.get("model", "")).lower() else "audio"
-        in_rate = audio_fmt["input"]["format"]["rate"]
-        out_rate = (audio_fmt.get("output") or {}).get("format", {}).get("rate", 24000)
-
-        session = {"type": args.type}
-        if args.type == "realtime":
-            instructions = args.instructions
-            if instructions is None:
-                instructions = TOOLS_INSTRUCTIONS if args.tools else MODE_INSTRUCTIONS[card].get(args.output_modalities)
-            if args.tools:
-                session["tools"] = DEMO_TOOLS
-            if instructions is not None:
-                session["instructions"] = instructions
-            if args.output_modalities:
-                session["output_modalities"] = args.output_modalities.split(",")
-        elif args.language:
-            session["audio"] = {"input": {"transcription": {"language": args.language}}}
-        await conn.session.update(session=session)  # ty: ignore[invalid-argument-type]
-        ev = await conn.recv()
-        if ev.type == "error":
-            print("error:", ev.error.message)
-            return
-
-        turns = [("audio", path) for path in args.wav] + [("text", text) for text in args.text]
-        for i, (kind, src) in enumerate(turns):
-            t_last = 0.0
-
-            # the input is sent while the events are read: live transcription arrives while the audio is sent
-            async def send_turn():
-                nonlocal t_last
-                if kind == "audio":
-                    pcm = read_wav(src, in_rate)
-                    step = in_rate * args.chunk_ms // 1000
-                    for j in range(0, len(pcm), step):
-                        chunk = pcm[j:j + step]
-                        await conn.input_audio_buffer.append(audio=base64.b64encode(chunk.tobytes()).decode())
-                        if args.realtime:
-                            await asyncio.sleep(len(chunk) / in_rate)
-                    if args.type == "transcription":
-                        # "final" ends the input, models that transcribe with a delay then complete the last item
-                        await conn.send({"type": "input_audio_buffer.commit", "final": i == len(args.wav) - 1})  # ty: ignore[invalid-argument-type]
-                else:
-                    await conn.conversation.item.create(item={
-                        "type": "message", "role": "user", "content": [{"type": "input_text", "text": src}]})
-                if args.type == "realtime":
-                    # the audio not committed yet ends the user turn
-                    await conn.response.create()
-                t_last = time.perf_counter()
-
-            print(f"--- turn {i} ({kind}: {src})")
-            sender = asyncio.create_task(send_turn())
-            t_first = None
-            audio = []
-            async for ev in conn:
-                if ev.type == "response.output_text.delta" or ev.type == "conversation.item.input_audio_transcription.delta":
-                    t_first = t_first or time.perf_counter()
-                    print(ev.delta, end="", flush=True)
-                elif ev.type == "response.output_audio.delta":
-                    t_first = t_first or time.perf_counter()
-                    audio.append(np.frombuffer(base64.b64decode(ev.delta), dtype=np.int16))
-                elif ev.type == "response.done":
-                    # the calls of the response are run, their results go back and the model answers them
-                    calls = [o for o in ev.model_dump()["response"]["output"] if o["type"] == "function_call"]
-                    for call in calls:
-                        result = json.dumps(run_tool(call["name"], call["arguments"]))
-                        print(f"\n[call {call['name']}({call['arguments']}) -> {result}]", flush=True)
-                        await conn.conversation.item.create(item={
-                            "type": "function_call_output", "call_id": call["call_id"], "output": result})
-                    if not calls:
-                        break
-                    await conn.response.create()
-                elif ev.type == "conversation.item.input_audio_transcription.completed":
-                    break
-                elif ev.type == "error":
-                    print("\nerror:", ev.error.message)
-                    return
-            await sender
-            print()
-            if t_first is not None and t_first > t_last:
-                print(f"first output {1000 * (t_first - t_last):.0f} ms after the end of the input")
-            elif t_first is not None:
-                print(f"output started {1000 * (t_last - t_first):.0f} ms before the end of the input, "
-                      f"done {1000 * (time.perf_counter() - t_last):.0f} ms after it")
-            if audio:
-                pcm_out = np.concatenate(audio)
-                path = f"{args.out.removesuffix('.wav')}_{i}.wav" if len(turns) > 1 else args.out
-                write_wav(path, pcm_out, out_rate)
-                print(f"audio: {len(pcm_out) / out_rate:.2f} s -> {path}")
-
-
-if __name__ == "__main__":
-    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("wav", nargs="*", help="PCM16 wav files, one user turn each")
-    p.add_argument("--text", action="append", default=[], help="a user turn as text, after the audio turns, repeatable")
-    p.add_argument("--type", choices=["realtime", "transcription"], default="realtime", help="session type")
-    p.add_argument("--instructions", help="system prompt of the conversation")
-    p.add_argument("--tools", action="store_true", help="give the conversation a demo tool (set_volume), its calls are run")
-    p.add_argument("--output-modalities", choices=["text", "audio", "text,audio"], help="default: the model picks")
-    p.add_argument("--language", help="transcription language, e.g. en")
-    p.add_argument("--chunk-ms", type=int, default=100, help="audio chunk size")
-    p.add_argument("--realtime", action=argparse.BooleanOptionalAction, default=True, help="send the audio in real time")
-    p.add_argument("--out", default="response.wav", help="where the audio of the responses goes")
-    p.add_argument("--url", default="http://127.0.0.1:9931/v1")
-    p.add_argument("--api-key", default="none")
-    p.add_argument("--model", default="default", help="model name, used by the router")
-    asyncio.run(main(p.parse_args()))

ggml/CMakeLists.txt

@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)
 
 ### GGML Version
 set(GGML_VERSION_MAJOR 0)
-set(GGML_VERSION_MINOR 29)
+set(GGML_VERSION_MINOR 27)
 set(GGML_VERSION_PATCH 0)
 set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
 
@@ -250,7 +250,6 @@ option(GGML_OPENMP_FETCH                    "ggml: fetch LLVM OpenMP"
 option(GGML_RPC                             "ggml: use RPC"                                   OFF)
 option(GGML_SYCL                            "ggml: use SYCL"                                  OFF)
 option(GGML_SYCL_F16                        "ggml: use 16 bit floats for sycl calculations"   OFF)
-option(GGML_SYCL_SUPPORT_XMX                "ggml: build the XMX (DPAS) code paths in the SYCL backend" ON)
 option(GGML_SYCL_GRAPH                      "ggml: enable graphs in the SYCL backend"         ON)
 option(GGML_SYCL_HOST_MEM_FALLBACK          "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON)
 option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API     "ggml: use Level Zero API in SYCL backend"  ON)

ggml/include/ggml-backend.h

@@ -70,8 +70,7 @@ extern "C" {
     GGML_API void                           ggml_backend_buffer_reset         (ggml_backend_buffer_t buffer);
 
     // tensor copy between different backends
-    GGML_API void ggml_backend_tensor_copy  (const struct ggml_tensor * src, struct ggml_tensor * dst);
-    GGML_API void ggml_backend_tensor_copy_n(const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors);
+    GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst);
 
     //
     // Backend (stream)
@@ -96,8 +95,6 @@ extern "C" {
     GGML_API void ggml_backend_tensor_get   (const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);
     GGML_API void ggml_backend_tensor_set_2d(      struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
     GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
-    GGML_API void ggml_backend_tensor_set_n (      struct ggml_tensor * const * tensors, const void * data, const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors);
-    GGML_API void ggml_backend_tensor_get_n (const struct ggml_tensor * const * tensors,       void * data, const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors);
     GGML_API void ggml_backend_tensor_memset(      struct ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);
 
     GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
@@ -211,18 +208,6 @@ extern "C" {
 
     // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend)
     typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends);
-
-    // Builds the communicator for the case where each rank is a separate process, so a process
-    // owns one backend of the communicator rather than all of them.
-    //
-    // The caller provides bootstrap_callback, which moves bytes between the ranks before the
-    // communicator exists. Each rank passes size bytes in send_buf, and recv_buf receives
-    // world_size * size bytes in rank order. The backend decides what to exchange and how many
-    // times to call it, and every rank must make the same calls in the same order.
-    // comm_init_rank blocks until all world_size ranks have joined.
-    typedef bool   (*ggml_backend_comm_bootstrap_callback)(void * user_data, const void * send_buf, void * recv_buf, size_t size);
-    typedef void * (*ggml_backend_comm_init_rank_t)(ggml_backend_t backend, int rank, int world_size, ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data);
-
     typedef void   (*ggml_backend_comm_free_t)(void * comm_ctx);
     typedef bool   (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);
 

ggml/include/ggml.h

@@ -381,8 +381,6 @@ extern "C" {
     GGML_API void        ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
     GGML_API void        ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
 
-    typedef struct { uint8_t bits; } ggml_fp8_e4m3_t;
-
     struct ggml_object;
     struct ggml_context;
     struct ggml_cgraph;
@@ -432,8 +430,7 @@ extern "C" {
         GGML_TYPE_NVFP4   = 40, // NVFP4 (4 blocks, E4M3 scale)
         GGML_TYPE_Q1_0    = 41,
         GGML_TYPE_Q2_0    = 42,
-        GGML_TYPE_F8_E4M3 = 43,
-        GGML_TYPE_COUNT   = 44,
+        GGML_TYPE_COUNT   = 43,
     };
 
     // [TAG_GGML_PREC]
@@ -489,7 +486,6 @@ extern "C" {
         GGML_FTYPE_MOSTLY_NVFP4   = 26, // except 1d tensors
         GGML_FTYPE_MOSTLY_Q1_0    = 27, // except 1d tensors
         GGML_FTYPE_MOSTLY_Q2_0    = 28, // except 1d tensors
-        GGML_FTYPE_MOSTLY_F8_E4M3 = 29, // except 1d tensors
     };
 
     // available tensor operations:
@@ -2516,24 +2512,6 @@ extern "C" {
             float                 max_bias,
             float                 logit_softcap);
 
-    // same as ggml_flash_attn_ext, but each q slice attends to its own list of K/V rows
-    // q:       [n_embd_k, n_batch, n_head,    ne3]
-    // k:       [n_embd_k, n_rows,  n_head_kv, 1  ] shared by all q slices
-    // v:       [n_embd_v, n_rows,  n_head_kv, 1  ]
-    // mask:    [n_kv,     n_batch, ne32,      ne3]
-    // kv_rows: [n_kv,     ne3] I32, mask column i of slice i3 uses row kv_rows[i, i3] of k and v
-    // negative rows are padding, their mask entries must be -inf
-    GGML_API struct ggml_tensor * ggml_flash_attn_ext_rows(
-            struct ggml_context * ctx,
-            struct ggml_tensor  * q,
-            struct ggml_tensor  * k,
-            struct ggml_tensor  * v,
-            struct ggml_tensor  * mask,
-            struct ggml_tensor  * kv_rows,
-            float                 scale,
-            float                 max_bias,
-            float                 logit_softcap);
-
     GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(
             struct ggml_tensor * a,
             enum ggml_prec       prec),

ggml/src/ggml-backend-impl.h

@@ -47,9 +47,6 @@ extern "C" {
     // Backend buffer
     //
 
-    // TODO: remove `set_tensor`, `get_tensor`, `set_tensor_2d`, `get_tensor_2d` and `cpy_tensor` from the buffer interface
-    //       they are superseded by the `set_tensor_n`, `get_tensor_n`, `cpy_tensor_n` interface
-    //       ref: https://github.com/ggml-org/llama.cpp/pull/30881#discussion_r4400299490
     struct ggml_backend_buffer_i {
         // (optional) free the buffer
         void         (*free_buffer)  (ggml_backend_buffer_t buffer);
@@ -71,11 +68,6 @@ extern "C" {
         void         (*clear)        (ggml_backend_buffer_t buffer, uint8_t value);
         // (optional) reset any internal state due to tensor initialization, such as tensor extras
         void         (*reset)        (ggml_backend_buffer_t buffer);
-        // (optional) batched tensor data copies
-        void         (*set_tensor_n) (ggml_backend_buffer_t buffer,       struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
-        void         (*get_tensor_n) (ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors,       void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
-        // (optional) batched tensor copies: returns true if all copies were performed, false if unsupported and no copies were performed
-        bool         (*cpy_tensor_n) (ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors);
     };
 
     struct ggml_backend_buffer {
@@ -92,9 +84,6 @@ extern "C" {
                    void *                     context,
                    size_t                     size);
 
-    // do not use directly, use ggml_backend_tensor_copy instead
-    GGML_API bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
-
     // multi-buffer
     // buffer that contains a collection of buffers
     GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);

ggml/src/ggml-backend-meta.cpp

@@ -489,32 +489,6 @@ static struct ggml_tensor * ggml_backend_meta_buffer_simple_tensor(const struct
     return it->second[index];
 }
 
-// bytes spanned by all dims except skip_dim
-static size_t ggml_backend_meta_inner_extent(const struct ggml_tensor * tensor, int skip_dim) {
-    size_t extent = 0;
-    for (int i = 0; i < GGML_MAX_DIMS; i++) {
-        if (i != skip_dim && tensor->ne[i] > 1) {
-            extent += (size_t) (tensor->ne[i] - 1) * tensor->nb[i];
-        }
-    }
-    return extent;
-}
-
-// the view dim that holds the split axis of src, or -1
-// all other dims of src and view must stay inside one slab of that axis
-static int ggml_backend_meta_view_slab_dim(const struct ggml_tensor * view, const struct ggml_tensor * src, int axis) {
-    const size_t slab = src->nb[axis];
-    if (ggml_backend_meta_inner_extent(src, axis) >= slab) {
-        return -1;
-    }
-    for (int dim = 0; dim < GGML_MAX_DIMS; dim++) {
-        if (view->ne[dim] == src->ne[axis] && view->nb[dim] == slab) {
-            return ggml_backend_meta_inner_extent(view, dim) < slab ? dim : -1;
-        }
-    }
-    return -1;
-}
-
 static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync);
 
 static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
@@ -726,39 +700,9 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
             }
             GGML_ABORT("fatal error");
         }
-        // permuted view: split the view dim that spans the src split axis, other dims must be inside one element of it or outside the span
-        if (ggml_is_permuted(tensor) && !ggml_is_permuted(tensor->src[0]) && axis >= 0 && axis < GGML_MAX_DIMS-1) {
-            const size_t nb_axis = tensor->src[0]->nb[axis];
-            const size_t nb_next = tensor->src[0]->nb[axis+1];
-            for (int dim = 0; dim < GGML_MAX_DIMS; dim++) {
-                if (tensor->ne[dim] == 1 || tensor->nb[dim] < nb_axis || tensor->nb[dim]*tensor->ne[dim] != nb_next) {
-                    continue;
-                }
-                bool others_ok = true;
-                for (int i = 0; i < GGML_MAX_DIMS; i++) {
-                    if (i == dim || tensor->ne[i] == 1) {
-                        continue;
-                    }
-                    others_ok = others_ok && (tensor->nb[i]*tensor->ne[i] <= tensor->nb[dim] || tensor->nb[i] % nb_next == 0);
-                }
-                if (others_ok) {
-                    return {ggml_backend_meta_split_axis(dim), {0}, {1}, 1};
-                }
-            }
-        }
         if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED || src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) {
             return src_ss[0];
         }
-        // a view that only regroups dims inside one slab of the split axis keeps the split,
-        // the per-device strides are only rescaled above nb[axis] (e.g. the KV streams of a permuted q)
-        if (axis >= 0 && axis < GGML_MAX_DIMS && tensor->view_offs == tensor->src[0]->view_offs) {
-            const int dim = ggml_backend_meta_view_slab_dim(tensor, tensor->src[0], axis);
-            if (dim >= 0) {
-                ggml_backend_meta_split_state ret = src_ss[0];
-                ret.axis = ggml_backend_meta_split_axis(dim);
-                return ret;
-            }
-        }
         GGML_ABORT("view of permuted tensor not implemented");
         //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1};
     };
@@ -1056,13 +1000,8 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
                 split_state = handle_pad(src_ss);
             } break;
             case GGML_OP_PAD_REFLECT_1D:
-            case GGML_OP_ROLL: {
-                split_state = handle_generic(src_ss, /*scalar_only =*/ true);
-            } break;
-            case GGML_OP_ARANGE: {
-                // no sources, every device computes the full range
-                split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1};
-            } break;
+            case GGML_OP_ROLL:
+            case GGML_OP_ARANGE:
             case GGML_OP_TIMESTEP_EMBEDDING: {
                 split_state = handle_generic(src_ss, /*scalar_only =*/ true);
             } break;
@@ -1760,9 +1699,6 @@ static const ggml_backend_buffer_i ggml_backend_meta_buffer_iface = {
     /* .cpy_tensor      = */ nullptr,
     /* .clear           = */ ggml_backend_meta_buffer_clear,
     /* .reset           = */ ggml_backend_meta_buffer_reset,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {

ggml/src/ggml-backend.cpp

@@ -353,7 +353,7 @@ void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {
     }
 }
 
-bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {
+static bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {
     ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;
     if (dst_buf->iface.cpy_tensor) {
         return dst_buf->iface.cpy_tensor(dst_buf, src, dst);
@@ -402,62 +402,6 @@ size_t ggml_backend_get_max_size(ggml_backend_t backend) {
     return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));
 }
 
-template <typename cb_get_buffer>
-static void ggml_backend_tensor_buft_groups(size_t n_tensors, cb_get_buffer && get_buffer, std::vector<std::vector<size_t>> & groups) {
-    std::unordered_map<ggml_backend_buffer_type_t, size_t> buft_idx;
-    groups.clear();
-    for (size_t i = 0; i < n_tensors; i++) {
-        ggml_backend_buffer_t buf = get_buffer(i);
-        auto it = buft_idx.find(buf->buft);
-        if (it == buft_idx.end()) {
-            buft_idx[buf->buft] = groups.size();
-            groups.emplace_back();
-        }
-        groups[buft_idx[buf->buft]].push_back(i);
-    }
-}
-
-static bool ggml_backend_tensor_batch_is_2d(
-        const struct ggml_tensor * const * tensors,
-        const size_t * offsets,
-        const size_t * sizes,
-        const size_t * data_offsets,
-        size_t n_tensors,
-        size_t * base_offset,
-        size_t * size,
-        size_t * stride_tensor,
-        size_t * stride_data) {
-    if (n_tensors <= 1) {
-        return false;
-    }
-
-    const struct ggml_tensor * tensor = tensors[0];
-    const size_t size0 = sizes[0];
-
-    if (size0 == 0 || offsets[1] < offsets[0] || data_offsets[1] < data_offsets[0]) {
-        return false;
-    }
-
-    const size_t stride_tensor0 = offsets[1] - offsets[0];
-    const size_t stride_data0   = data_offsets[1] - data_offsets[0];
-
-    for (size_t i = 1; i < n_tensors; i++) {
-        if (tensors[i] != tensor || sizes[i] != size0) {
-            return false;
-        }
-        if (offsets[i] != offsets[0] + i*stride_tensor0 || data_offsets[i] != data_offsets[0] + i*stride_data0) {
-            return false;
-        }
-    }
-
-    *base_offset   = offsets[0];
-    *size          = size0;
-    *stride_tensor = stride_tensor0;
-    *stride_data   = stride_data0;
-
-    return true;
-}
-
 void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
     GGML_ASSERT(backend);
     GGML_ASSERT(tensor);
@@ -602,154 +546,6 @@ void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data,
     buf->iface.get_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
 }
 
-void ggml_backend_tensor_set_n(struct ggml_tensor * const * tensors, const void * data,
-        const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors) {
-    if (n_tensors == 0) {
-        return;
-    }
-
-    std::vector<const void *> data_ptrs(n_tensors);
-    for (size_t i = 0; i < n_tensors; i++) {
-        data_ptrs[i] = (const char *) data + data_offsets[i];
-    }
-
-    ggml_backend_buffer_t buf = NULL;
-    bool same_buft = true;
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        GGML_ASSERT(tensors[i] != NULL);
-        ggml_backend_buffer_t tensor_buf = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
-        GGML_ASSERT(tensor_buf != NULL && "tensor buffer not set");
-        GGML_ASSERT(tensors[i]->data != NULL && "tensor not allocated");
-        GGML_ASSERT(offsets[i] + sizes[i] <= ggml_nbytes(tensors[i]) && "tensor write out of bounds");
-
-        if (buf == NULL) {
-            buf = tensor_buf;
-        } else if (buf->buft != tensor_buf->buft) {
-            same_buft = false;
-        }
-    }
-
-    if (same_buft && buf->iface.set_tensor_n != NULL) {
-        buf->iface.set_tensor_n(buf, tensors, data_ptrs.data(), offsets, sizes, n_tensors);
-        return;
-    }
-
-    if (!same_buft) {
-        std::vector<std::vector<size_t>> groups;
-        ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) {
-            return tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
-        }, groups);
-
-        for (size_t g = 0; g < groups.size(); g++) {
-            std::vector<ggml_tensor *> g_tensors;
-            std::vector<size_t> g_offsets;
-            std::vector<size_t> g_sizes;
-            std::vector<size_t> g_data_offsets;
-            g_tensors.reserve(groups[g].size());
-            g_offsets.reserve(groups[g].size());
-            g_sizes.reserve(groups[g].size());
-            g_data_offsets.reserve(groups[g].size());
-
-            for (size_t i : groups[g]) {
-                g_tensors.push_back(tensors[i]);
-                g_offsets.push_back(offsets[i]);
-                g_sizes.push_back(sizes[i]);
-                g_data_offsets.push_back(data_offsets[i]);
-            }
-
-            ggml_backend_tensor_set_n(g_tensors.data(), data, g_offsets.data(), g_sizes.data(), g_data_offsets.data(), g_tensors.size());
-        }
-        return;
-    }
-
-    size_t base_offset = 0, size = 0, stride_tensor = 0, stride_data = 0;
-    if (ggml_backend_tensor_batch_is_2d(tensors, offsets, sizes, data_offsets, n_tensors,
-                &base_offset, &size, &stride_tensor, &stride_data)) {
-        ggml_backend_tensor_set_2d(tensors[0], data_ptrs[0], base_offset, size,
-                n_tensors, stride_tensor, stride_data);
-        return;
-    }
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        ggml_backend_tensor_set(tensors[i], data_ptrs[i], offsets[i], sizes[i]);
-    }
-}
-
-void ggml_backend_tensor_get_n(const struct ggml_tensor * const * tensors, void * data,
-        const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors) {
-    if (n_tensors == 0) {
-        return;
-    }
-
-    std::vector<void *> data_ptrs(n_tensors);
-    for (size_t i = 0; i < n_tensors; i++) {
-        data_ptrs[i] = (char *) data + data_offsets[i];
-    }
-
-    ggml_backend_buffer_t buf = NULL;
-    bool same_buft = true;
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        GGML_ASSERT(tensors[i] != NULL);
-        ggml_backend_buffer_t tensor_buf = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
-        GGML_ASSERT(tensor_buf != NULL && "tensor buffer not set");
-        GGML_ASSERT(tensors[i]->data != NULL && "tensor not allocated");
-        GGML_ASSERT(offsets[i] + sizes[i] <= ggml_nbytes(tensors[i]) && "tensor read out of bounds");
-
-        if (buf == NULL) {
-            buf = tensor_buf;
-        } else if (buf->buft != tensor_buf->buft) {
-            same_buft = false;
-        }
-    }
-
-    if (same_buft && buf->iface.get_tensor_n != NULL) {
-        buf->iface.get_tensor_n(buf, tensors, data_ptrs.data(), offsets, sizes, n_tensors);
-        return;
-    }
-
-    if (!same_buft) {
-        std::vector<std::vector<size_t>> groups;
-        ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) {
-            return tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
-        }, groups);
-
-        for (size_t g = 0; g < groups.size(); g++) {
-            std::vector<const ggml_tensor *> g_tensors;
-            std::vector<size_t> g_offsets;
-            std::vector<size_t> g_sizes;
-            std::vector<size_t> g_data_offsets;
-            g_tensors.reserve(groups[g].size());
-            g_offsets.reserve(groups[g].size());
-            g_sizes.reserve(groups[g].size());
-            g_data_offsets.reserve(groups[g].size());
-
-            for (size_t i : groups[g]) {
-                g_tensors.push_back(tensors[i]);
-                g_offsets.push_back(offsets[i]);
-                g_sizes.push_back(sizes[i]);
-                g_data_offsets.push_back(data_offsets[i]);
-            }
-
-            ggml_backend_tensor_get_n(g_tensors.data(), data, g_offsets.data(), g_sizes.data(), g_data_offsets.data(), g_tensors.size());
-        }
-        return;
-    }
-
-    size_t base_offset = 0, size = 0, stride_tensor = 0, stride_data = 0;
-    if (ggml_backend_tensor_batch_is_2d(tensors, offsets, sizes, data_offsets, n_tensors,
-                &base_offset, &size, &stride_tensor, &stride_data)) {
-        ggml_backend_tensor_get_2d(tensors[0], data_ptrs[0], base_offset, size,
-                n_tensors, stride_tensor, stride_data);
-        return;
-    }
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        ggml_backend_tensor_get(tensors[i], data_ptrs[i], offsets[i], sizes[i]);
-    }
-}
-
 void ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
     GGML_ASSERT(tensor);
     ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
@@ -852,59 +648,6 @@ void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor
     }
 }
 
-void ggml_backend_tensor_copy_n(const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) {
-    if (n_tensors == 0) {
-        return;
-    }
-
-    for (size_t i = 0; i < n_tensors; ++i) {
-        GGML_ASSERT(ggml_are_same_layout(srcs[i], dsts[i]) && "cannot copy tensors with different layouts");
-    }
-
-    ggml_backend_buffer_t buf = dsts[0]->view_src ? dsts[0]->view_src->buffer : dsts[0]->buffer;
-    bool same_buft = true;
-    for (size_t i = 0; i < n_tensors; ++i) {
-        ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
-        if (dst_buf->buft != buf->buft) {
-            same_buft = false;
-            break;
-        }
-    }
-
-    // if cpy_tensor_n returns false, it must not have performed any copies
-    if (same_buft && buf->iface.cpy_tensor_n != NULL) {
-        if (buf->iface.cpy_tensor_n(buf, srcs, dsts, n_tensors)) {
-            return;
-        }
-    }
-
-    if (!same_buft) {
-        std::vector<std::vector<size_t>> groups;
-        ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) {
-            return dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
-        }, groups);
-
-        for (size_t g = 0; g < groups.size(); g++) {
-            std::vector<const ggml_tensor *> g_srcs;
-            std::vector<ggml_tensor *> g_dsts;
-            g_srcs.reserve(groups[g].size());
-            g_dsts.reserve(groups[g].size());
-
-            for (size_t i : groups[g]) {
-                g_srcs.push_back(srcs[i]);
-                g_dsts.push_back(dsts[i]);
-            }
-
-            ggml_backend_tensor_copy_n(g_srcs.data(), g_dsts.data(), g_srcs.size());
-        }
-        return;
-    }
-
-    for (size_t i = 0; i < n_tensors; ++i) {
-        ggml_backend_tensor_copy(srcs[i], dsts[i]);
-    }
-}
-
 void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
     GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
 
@@ -1110,9 +853,6 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
     /* .cpy_tensor      = */ NULL,
     /* .clear           = */ ggml_backend_multi_buffer_clear,
     /* .reset           = */ NULL,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
@@ -2792,9 +2532,6 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
     /* .cpy_tensor      = */ ggml_backend_cpu_buffer_cpy_tensor,
     /* .clear           = */ ggml_backend_cpu_buffer_clear,
     /* .reset           = */ NULL,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
@@ -2809,9 +2546,6 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
     /* .cpy_tensor      = */ ggml_backend_cpu_buffer_cpy_tensor,
     /* .clear           = */ ggml_backend_cpu_buffer_clear,
     /* .reset           = */ NULL,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 // CPU backend buffer type

ggml/src/ggml-cann/ggml-cann.cpp

@@ -1478,9 +1478,6 @@ static const ggml_backend_buffer_i ggml_backend_cann_buffer_interface = {
     /* .cpy_tensor      = */ ggml_backend_cann_buffer_cpy_tensor,
     /* .clear           = */ ggml_backend_cann_buffer_clear,
     /* .reset           = */ NULL,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 /**
@@ -2670,10 +2667,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
             return true;
         case GGML_OP_FLASH_ATTN_EXT:
             {
-                // K/V row lists (ggml_flash_attn_ext_rows) are not supported
-                if (op->src[5] != nullptr) {
-                    return false;
-                }
 #ifdef ASCEND_310P
                 // FA not support on 310p device
                 return false;

ggml/src/ggml-cpu/amx/amx.cpp

@@ -116,9 +116,6 @@ static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {
     /* .cpy_tensor      = */ nullptr,
     /* .clear           = */ ggml_backend_amx_buffer_clear,
     /* .reset           = */ nullptr,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 static const char * ggml_backend_amx_buffer_type_get_name(ggml_backend_buffer_type_t buft) {

ggml/src/ggml-cpu/arch/arm/repack.cpp

@@ -535,60 +535,6 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v
     UNUSED(ncols_interleaved);
     UNUSED(blocklen);
 
-#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
-    if (svcntb() * 8 == 256) {
-        const svbool_t  pg16 = svptrue_pat_b8(SV_VL16);
-        const svbool_t  pg4  = svptrue_pat_b32(SV_VL4);
-        const svbool_t  pgb  = svptrue_b8();
-        const svbool_t  pgw  = svptrue_b32();
-        const svbool_t  pg_hi4 = svnot_b_z(pgw, pg4);
-        const svuint8_t m4b = svdup_n_u8(0x0f);
-        const svuint8_t values = svld1_u8(pg16, (const uint8_t *) kvalues_mxfp4);
-
-        // Reorder Q8 four-byte groups as [0,2,4,6,1,3,5,7]
-        const svuint32_t q8_group_index = svsub_n_u32_m(pg_hi4, svindex_u32(0, 2), 7);
-
-        const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
-        float * res_ptr = s;
-
-        for (int x = 0; x < nc / ncols_interleaved; x++) {
-            const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);
-            svfloat32_t sumf = svdup_n_f32(0.0f);
-
-            for (int l = 0; l < nb; l++) {
-                const svint8_t q8 = svld1_s8(pgb, a_ptr[l].qs);
-                const svint8_t q8_even_odd = svreinterpret_s8_u32(svtbl_u32(svreinterpret_u32_s8(q8), q8_group_index));
-                const svuint8_t qmx01 = svld1_u8(pgb, b_ptr[l].qs + 0);
-                const svuint8_t qmx_lo01 = svtbl_u8(values, svand_u8_x(pgb, qmx01, m4b));
-                const svuint8_t qmx_hi01 = svtbl_u8(values, svlsr_n_u8_x(pgb, qmx01, 4));
-                const svuint8_t qmx23 = svld1_u8(pgb, b_ptr[l].qs + 32);
-                const svuint8_t qmx_lo23 = svtbl_u8(values, svand_u8_x(pgb, qmx23, m4b));
-                const svuint8_t qmx_hi23 = svtbl_u8(values, svlsr_n_u8_x(pgb, qmx23, 4));
-
-                svint32_t sumi = svdup_n_s32(0);
-                sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_lo01), q8_even_odd, 0);
-                sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_lo23), q8_even_odd, 1);
-                sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_hi01), q8_even_odd, 2);
-                sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_hi23), q8_even_odd, 3);
-                const svint32_t sumi_cols = svadd_s32_z(pg4, sumi, svext_s32(sumi, sumi, 4));
-
-                // Vector form of GGML_CPU_E8M0_TO_FP32_HALF
-                const svuint32_t e_u32 = svld1ub_u32(pg4, b_ptr[l].e);
-                const svbool_t e_is_denormal = svcmplt_n_u32(pg4, e_u32, 2);
-                const svuint32_t e_denormal_bits = svlsl_u32_x(pg4, svdup_n_u32(0x00200000), e_u32);
-                const svuint32_t e_normal_bits = svlsl_n_u32_x(pg4, svsub_n_u32_x(pg4, e_u32, 1), 23);
-                const svuint32_t e_bits = svsel_u32(e_is_denormal, e_denormal_bits, e_normal_bits);
-                const svfloat16_t d_f16 = svreinterpret_f16_u16(svdup_n_u16(a_ptr[l].d));
-                const svfloat32_t d_f32 = svcvt_f32_f16_x(pg4, d_f16);
-                const svfloat32_t scale = svmul_f32_x(pg4, svreinterpret_f32_u32(e_bits), d_f32);
-                sumf = svmla_f32_m(pg4, sumf,svcvt_f32_s32_m(svdup_n_f32(0.0f), pg4, sumi_cols), scale);
-            }
-            svst1_f32(pg4, res_ptr + x * 4, sumf);
-        }
-        return;
-    }
-#endif // defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
-
 #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
     const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);
     const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
@@ -1845,71 +1791,6 @@ void ggml_gemv_q8_0_4x8_q8_0(int                        n,
     UNUSED(ncols_interleaved);
     UNUSED(blocklen);
 
-#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
-    // 256-bit SVE: 32 x int8 -> 8 x int32
-    if (ggml_cpu_get_sve_cnt() == QK8_0) {
-        const block_q8_0x4 * GGML_RESTRICT b_ptr = (const block_q8_0x4 *) vx;
-
-        const svbool_t pg   = svptrue_b8();
-        const svbool_t pg4w = svptrue_pat_b32(SV_VL4);
-
-        // after svtbl + svcvt the scale lanes are [d0,d0,d1,d1,d2,d2,d3,d3]: lane pair j = row j
-        const svuint16_t d_idx = svreinterpret_u16_u32(svlsr_n_u32_x(pg, svindex_u32(0, 1), 1));
-
-        for (int c = 0; c < nc; c += ncols_interleaved) {
-            const block_q8_0 * GGML_RESTRICT a_ptr = (const block_q8_0 *) vy;
-
-            svfloat32_t acc = svdup_n_f32(0.0f);
-
-            for (int b = 0; b < nb; ++b) {
-                const int8_t * GGML_RESTRICT bq = (const int8_t *) b_ptr->qs;
-
-                uint64_t a_chunk0;
-                uint64_t a_chunk1;
-                uint64_t a_chunk2;
-                uint64_t a_chunk3;
-                uint64_t b_d;
-
-                memcpy(&a_chunk0, a_ptr->qs +  0, sizeof(a_chunk0));
-                memcpy(&a_chunk1, a_ptr->qs +  8, sizeof(a_chunk1));
-                memcpy(&a_chunk2, a_ptr->qs + 16, sizeof(a_chunk2));
-                memcpy(&a_chunk3, a_ptr->qs + 24, sizeof(a_chunk3));
-                memcpy(&b_d, b_ptr->d, sizeof(b_d));
-
-                const svint8_t aq0 = svreinterpret_s8_u64(svdup_n_u64(a_chunk0));
-                const svint8_t aq1 = svreinterpret_s8_u64(svdup_n_u64(a_chunk1));
-                const svint8_t aq2 = svreinterpret_s8_u64(svdup_n_u64(a_chunk2));
-                const svint8_t aq3 = svreinterpret_s8_u64(svdup_n_u64(a_chunk3));
-
-                const svint8_t bq0 = svld1_s8(pg, bq +  0);
-                const svint8_t bq1 = svld1_s8(pg, bq + 32);
-                const svint8_t bq2 = svld1_s8(pg, bq + 64);
-                const svint8_t bq3 = svld1_s8(pg, bq + 96);
-
-                const svint32_t iacc0 = svdot_s32(svdup_n_s32(0), bq0, aq0);
-                const svint32_t iacc1 = svdot_s32(svdup_n_s32(0), bq1, aq1);
-                const svint32_t iacc2 = svdot_s32(svdup_n_s32(0), bq2, aq2);
-                const svint32_t iacc3 = svdot_s32(svdup_n_s32(0), bq3, aq3);
-
-                const svint32_t isum = svadd_s32_x(pg, svadd_s32_x(pg, iacc0, iacc1), svadd_s32_x(pg, iacc2, iacc3));
-
-                const svfloat16_t bd_h  = svtbl_f16(svreinterpret_f16_u64(svdup_n_u64(b_d)), d_idx);
-                const svfloat32_t scale = svmul_n_f32_x(pg, svcvt_f32_f16_x(pg, bd_h), GGML_CPU_FP16_TO_FP32(a_ptr->d));
-
-                acc = svmla_f32_x(pg, acc, svcvt_f32_s32_x(pg, isum), scale);
-
-                ++a_ptr;
-                ++b_ptr;
-            }
-
-            svst1_f32(pg4w, s, svadd_f32_x(pg, svuzp1_f32(acc, acc), svuzp2_f32(acc, acc)));
-            s += ncols_interleaved;
-        }
-
-        return;
-    }
-#endif  // defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
-
 #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
     const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx;
 
@@ -4993,7 +4874,6 @@ void ggml_gemm_q6_K_8x8_q8_K(int                        n,
 
     constexpr int ncols_interleaved = 8;
     constexpr int blocklen          = 8;
-    constexpr int q8_k_blocklen     = 4;
 
     assert(n % qk == 0);
     assert(nr % 4 == 0);
@@ -5002,195 +4882,9 @@ void ggml_gemm_q6_K_8x8_q8_K(int                        n,
     UNUSED(nb);
     UNUSED(ncols_interleaved);
     UNUSED(blocklen);
-    UNUSED(q8_k_blocklen);
-
-#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
-    if (svcntb() * 8 == 256) {
-        const svbool_t pg8  = svptrue_b8();
-        const svbool_t pg32 = svptrue_b32();
-
-        const svuint8_t low_nibble = svdup_n_u8(0x0f);
-        const svuint8_t high_lo2   = svdup_n_u8(0x03);
-        const svuint8_t high_hi2   = svdup_n_u8(0x30);
-        const svint8_t  q6_bias    = svdup_n_s8(32);
-        const svint32_t zero_s32   = svdup_n_s32(0);
-
-        for (int y = 0; y < nr / q8_k_blocklen; ++y) {
-            const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + y * nb;
-
-            for (int x = 0; x < nc / ncols_interleaved; ++x) {
-                const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + x * nb;
-
-                // one accumlator per output row
-                svfloat32_t acc_f32_r0 = svdup_n_f32(0);
-                svfloat32_t acc_f32_r1 = svdup_n_f32(0);
-                svfloat32_t acc_f32_r2 = svdup_n_f32(0);
-                svfloat32_t acc_f32_r3 = svdup_n_f32(0);
-
-                for (int b = 0; b < nb; ++b) {
-                    svint32_t acc_c03_r01 = zero_s32;
-                    svint32_t acc_c03_r23 = zero_s32;
-                    svint32_t acc_c47_r01 = zero_s32;
-                    svint32_t acc_c47_r23 = zero_s32;
-
-                    for (int half = 0; half < 2; ++half) {
-                        const uint8_t * ql_base = q6_ptr[b].ql + half * 512;
-                        const uint8_t * qh_base = q6_ptr[b].qh + half * 256;
-
-                        for (int sb = 0; sb < QK_K / 64; ++sb) {
-                            const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;
-                            const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;
-
-                            // load 16 bytes (8B of R0, 8B of R1) and repeat in upper half
-                            // low lanes will multiply with 16 bytes of C0 and high lanes will multiply with 16 bytes of C1
-                            const svint8_t q8_l_r01_k0 = svld1rq_s8(pg8, q8_base_l + 0);
-                            const svint8_t q8_l_r01_k1 = svld1rq_s8(pg8, q8_base_l + 32);
-                            const svint8_t q8_l_r23_k0 = svld1rq_s8(pg8, q8_base_l + 16);
-                            const svint8_t q8_l_r23_k1 = svld1rq_s8(pg8, q8_base_l + 48);
-
-                            const svint8_t q8_h_r01_k0 = svld1rq_s8(pg8, q8_base_h + 0);
-                            const svint8_t q8_h_r01_k1 = svld1rq_s8(pg8, q8_base_h + 32);
-                            const svint8_t q8_h_r23_k0 = svld1rq_s8(pg8, q8_base_h + 16);
-                            const svint8_t q8_h_r23_k1 = svld1rq_s8(pg8, q8_base_h + 48);
-
-                            const int ql_off = sb * QK_K / 2;
-                            const int qh_off = ql_off & 255;
-
-                            const int scale_idx_l = half * 8 + sb;
-                            const int scale_idx_h = scale_idx_l + 4;
-
-                            const svint32_t scales_l_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_l * 8);
-                            const svint32_t scales_h_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_h * 8);
-
-                            const svint32_t scales_l_c03 = svzip1_s32(scales_l_all, scales_l_all);
-                            const svint32_t scales_h_c03 = svzip1_s32(scales_h_all, scales_h_all);
-
-                            {
-                                svuint8_t q6_ql0_01 = svld1_u8(pg8, ql_base + ql_off);
-                                svuint8_t q6_ql1_01 = svld1_u8(pg8, ql_base + ql_off + 64);
-                                svuint8_t q6_qh0_01 = svld1_u8(pg8, qh_base + qh_off);
-                                svuint8_t q6_qh1_01 = svld1_u8(pg8, qh_base + qh_off + 64);
-
-                                if (sb > 1) {
-                                    q6_qh0_01 = svlsr_n_u8_x(pg8, q6_qh0_01, 2);
-                                    q6_qh1_01 = svlsr_n_u8_x(pg8, q6_qh1_01, 2);
-                                }
-
-                                const svuint8_t q6_qh0_01_hi = svand_u8_x(pg8, q6_qh0_01, high_hi2);
-                                const svuint8_t q6_qh1_01_hi = svand_u8_x(pg8, q6_qh1_01, high_hi2);
-
-                                const svuint8_t q6_l0_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_01, high_lo2), 4));
-                                const svuint8_t q6_l1_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_01, high_lo2), 4));
-                                const svuint8_t q6_h0_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_01, 4), q6_qh0_01_hi);
-                                const svuint8_t q6_h1_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_01, 4), q6_qh1_01_hi);
-
-                                const svint8_t q6_l0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_01_bits), q6_bias);
-                                const svint8_t q6_l1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_01_bits), q6_bias);
-                                const svint8_t q6_h0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_01_bits), q6_bias);
-                                const svint8_t q6_h1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_01_bits), q6_bias);
-
-                                svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r01_k0);
-                                dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_01, q8_l_r01_k1);
-
-                                svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r01_k0);
-                                dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_01, q8_h_r01_k1);
-
-                                svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r23_k0);
-                                dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_01, q8_l_r23_k1);
-
-                                svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r23_k0);
-                                dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_01, q8_h_r23_k1);
-
-                                acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_l_r01, scales_l_c03);
-                                acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_h_r01, scales_h_c03);
-                                acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_l_r23, scales_l_c03);
-                                acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_h_r23, scales_h_c03);
-                            }
-
-                            const svint32_t scales_l_c47 = svzip2_s32(scales_l_all, scales_l_all);
-                            const svint32_t scales_h_c47 = svzip2_s32(scales_h_all, scales_h_all);
-
-                            {
-                                svuint8_t q6_ql0_23 = svld1_u8(pg8, ql_base + ql_off + 32);
-                                svuint8_t q6_ql1_23 = svld1_u8(pg8, ql_base + ql_off + 96);
-                                svuint8_t q6_qh0_23 = svld1_u8(pg8, qh_base + qh_off + 32);
-                                svuint8_t q6_qh1_23 = svld1_u8(pg8, qh_base + qh_off + 96);
-
-                                if (sb > 1) {
-                                    q6_qh0_23 = svlsr_n_u8_x(pg8, q6_qh0_23, 2);
-                                    q6_qh1_23 = svlsr_n_u8_x(pg8, q6_qh1_23, 2);
-                                }
-
-                                const svuint8_t q6_qh0_23_hi = svand_u8_x(pg8, q6_qh0_23, high_hi2);
-                                const svuint8_t q6_qh1_23_hi = svand_u8_x(pg8, q6_qh1_23, high_hi2);
-
-                                const svuint8_t q6_l0_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_23, high_lo2), 4));
-                                const svuint8_t q6_l1_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_23, high_lo2), 4));
-                                const svuint8_t q6_h0_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_23, 4), q6_qh0_23_hi);
-                                const svuint8_t q6_h1_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_23, 4), q6_qh1_23_hi);
-
-                                const svint8_t q6_l0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_23_bits), q6_bias);
-                                const svint8_t q6_l1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_23_bits), q6_bias);
-                                const svint8_t q6_h0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_23_bits), q6_bias);
-                                const svint8_t q6_h1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_23_bits), q6_bias);
-
-                                svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r01_k0);
-                                dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_23, q8_l_r01_k1);
-
-                                svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r01_k0);
-                                dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_23, q8_h_r01_k1);
-
-                                svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r23_k0);
-                                dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_23, q8_l_r23_k1);
-
-                                svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r23_k0);
-                                dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_23, q8_h_r23_k1);
-
-                                acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_l_r01, scales_l_c47);
-                                acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_h_r01, scales_h_c47);
-                                acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_l_r23, scales_l_c47);
-                                acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_h_r23, scales_h_c47);
-                            }
-                        }
-                    }
-
-                    // acc_c03_r01 -> c0-r0,c0-r1,c1-r0,c1-r1,c2-r0,c2-r1,c3-r0,c3-r1
-                    // acc_c47_r01 -> c4-r0,c4-r1,c5-r0,c5-r1,c6-r0,c6-r1,c7-r0,c7-r1
-                    // svuzp1_s32 picks even lanes -> c0-r0,c1-r0,c2-r0,c3-r0,c4-r0,c5-r0,c6-r0,c7-r0
-                    // similarly svuzp2_s32 picks remaining values from odd lanes
-                    const svint32_t dot_r0 = svuzp1_s32(acc_c03_r01, acc_c47_r01);
-                    const svint32_t dot_r1 = svuzp2_s32(acc_c03_r01, acc_c47_r01);
-                    const svint32_t dot_r2 = svuzp1_s32(acc_c03_r23, acc_c47_r23);
-                    const svint32_t dot_r3 = svuzp2_s32(acc_c03_r23, acc_c47_r23);
-
-                    const svuint32_t q6_d_bits = svld1uh_u32(pg32, (const uint16_t *) q6_ptr[b].d);
-                    const svfloat32_t q6_d_f32 = svcvt_f32_f16_x(pg32, svreinterpret_f16_u32(q6_d_bits));
-
-                    const svfloat32_t scale_r0 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[0]);
-                    const svfloat32_t scale_r1 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[1]);
-                    const svfloat32_t scale_r2 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[2]);
-                    const svfloat32_t scale_r3 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[3]);
-
-                    acc_f32_r0 = svmla_f32_x(pg32, acc_f32_r0, svcvt_f32_s32_x(pg32, dot_r0), scale_r0);
-                    acc_f32_r1 = svmla_f32_x(pg32, acc_f32_r1, svcvt_f32_s32_x(pg32, dot_r1), scale_r1);
-                    acc_f32_r2 = svmla_f32_x(pg32, acc_f32_r2, svcvt_f32_s32_x(pg32, dot_r2), scale_r2);
-                    acc_f32_r3 = svmla_f32_x(pg32, acc_f32_r3, svcvt_f32_s32_x(pg32, dot_r3), scale_r3);
-                }
-
-                const int row = y * q8_k_blocklen;
-                const int col = x * ncols_interleaved;
-
-                svst1_f32(pg32, s + (row + 0) * bs + col, acc_f32_r0);
-                svst1_f32(pg32, s + (row + 1) * bs + col, acc_f32_r1);
-                svst1_f32(pg32, s + (row + 2) * bs + col, acc_f32_r2);
-                svst1_f32(pg32, s + (row + 3) * bs + col, acc_f32_r3);
-            }
-        }
-        return;
-    }
-#endif  // SVE compile-time end
 
 #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+    constexpr int    q8_k_blocklen = 4;
     const uint8x16_t m4b           = vdupq_n_u8(0x0f);
     const uint8x16_t mask_lo       = vdupq_n_u8(0x03);
     const uint8x16_t mask_hi       = vdupq_n_u8(0x30);

ggml/src/ggml-cpu/arch/riscv/repack.cpp

@@ -29,7 +29,7 @@ void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTR
     assert(k % QK8_0 == 0);
     const int nb = k / QK8_0;
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
     block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;
     const size_t vl_calc = __riscv_vsetvl_e32m8(QK8_0);
     const size_t vl_save = __riscv_vsetvl_e64m2(4);

ggml/src/ggml-cpu/common.h

@@ -43,14 +43,6 @@ static inline float f32_to_f32(float x) {
     return x;
 }
 
-static inline float f8_e4m3_to_f32(ggml_fp8_e4m3_t x) {
-    return ggml_f8_e4m3_to_fp32(x.bits);
-}
-
-static inline ggml_fp8_e4m3_t f32_to_f8_e4m3(float x) {
-    return { ggml_fp32_to_f8_e4m3(x) };
-}
-
 // TODO - merge this into the traits table, after using row-based conversions
 template <class T>
 struct type_conversion_table;
@@ -73,12 +65,6 @@ struct type_conversion_table<ggml_bf16_t> {
     static constexpr ggml_bf16_t (*from_f32)(float) = f32_to_bf16;
 };
 
-template <>
-struct type_conversion_table<ggml_fp8_e4m3_t> {
-    static constexpr float (*to_f32)(ggml_fp8_e4m3_t) = f8_e4m3_to_f32;
-    static constexpr ggml_fp8_e4m3_t (*from_f32)(float) = f32_to_f8_e4m3;
-};
-
 template <>
 struct type_conversion_table<int32_t> {
     static constexpr float (*to_f32)(int32_t) = i32_to_f32;

ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -30,7 +30,7 @@ struct ggml_compute_params {
 };
 
 
-#if defined(_MSC_VER) && !defined(__clang__)
+#if defined(_MSC_VER)
 
 #define m512bh(p) p
 #define m512i(p) p
@@ -345,7 +345,7 @@ static inline int32x4_t ggml_nvfp4_dot8(const int8x8_t q4_lo, const int8x8_t q8_
 #include <immintrin.h>
 #endif
 
-#if defined(__riscv_v)
+#ifdef __riscv_v_intrinsic
 #include <riscv_vector.h>
 #endif
 

ggml/src/ggml-cpu/ggml-cpu.c

@@ -86,9 +86,6 @@ float ggml_table_f32_e8m0_half[1 << 8];
 // precomputed f32 table for ue4m3 (1 KB) (simd-mappings.h)
 float ggml_table_f32_ue4m3[1 << 8];
 
-// Signed FP8 needs its own table: UE4M3 ignores the sign, halves values for NVFP4, and maps NaN to zero.
-float ggml_table_f32_f8_e4m3[1 << 8];
-
 #if defined(__ARM_ARCH)
 struct ggml_arm_arch_features_type {
     int sve_cnt;
@@ -299,12 +296,6 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
         .vec_dot_type             = GGML_TYPE_Q8_0,
         .nrows                    = 1,
     },
-    [GGML_TYPE_F8_E4M3] = {
-        .from_float               = quantize_row_f8_e4m3,
-        .vec_dot                  = ggml_vec_dot_f8_e4m3_f32,
-        .vec_dot_type             = GGML_TYPE_F32,
-        .nrows                    = 1,
-    },
     [GGML_TYPE_Q2_K] = {
         .from_float               = quantize_row_q2_K,
         .vec_dot                  = ggml_vec_dot_q2_K_q8_K,
@@ -746,7 +737,7 @@ static void ggml_init_arm_arch_features(void) {}
 #endif
 #endif // __ARM_ARCH
 
-#if defined(__riscv) && defined(__riscv_v)
+#if defined(__riscv) && defined(__riscv_v_intrinsic)
 #include <riscv_vector.h>
 static void ggml_init_riscv_arch_features(void) {
     ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb();
@@ -2885,7 +2876,7 @@ struct ggml_cplan ggml_graph_plan(
                 case GGML_OP_ADD_ID:
                 case GGML_OP_ADD1:
                     {
-                        if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3) {
+                        if (ggml_is_quantized(node->src[0]->type)) {
                             cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
                         }
                     } break;
@@ -2933,7 +2924,7 @@ struct ggml_cplan ggml_graph_plan(
                     } break;
                 case GGML_OP_OUT_PROD:
                     {
-                        if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3 ||
+                        if (ggml_is_quantized(node->src[0]->type) ||
                             node->src[0]->type == GGML_TYPE_F16) {
                             cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
                         }
@@ -3556,7 +3547,7 @@ void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) {
         _mm_storeu_ps(y + i, y_vec);
     }
 
-#elif defined(__riscv_v) && defined(__riscv_zvfhmin)
+#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin)
     // calculate step size
     const int epr = __riscv_vsetvlmax_e16m2();
     const int step = epr * 2;
@@ -3626,7 +3617,7 @@ void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) {
                                         (const __m128i *)(x + i))),
                                 16)));
     }
-#elif defined(__riscv_v) && defined(__riscv_zvfbfmin)
+#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin)
     // calculate step size
     const int epr = __riscv_vsetvlmax_e16m2();
     const int step = epr * 2;
@@ -3746,7 +3737,7 @@ int ggml_cpu_has_arm_fma(void) {
 }
 
 int ggml_cpu_has_riscv_v(void) {
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
     return 1;
 #else
     return 0;
@@ -3754,7 +3745,7 @@ int ggml_cpu_has_riscv_v(void) {
 }
 
 int ggml_cpu_get_rvv_vlen(void) {
-#if defined(__riscv) && defined(__riscv_v)
+#if defined(__riscv) && defined(__riscv_v_intrinsic)
     return ggml_riscv_arch_features.rvv_vlen;
 #else
     return 0;
@@ -3917,7 +3908,6 @@ void ggml_cpu_init(void) {
             // initialize UE4M3 table (256 entries)
             for (int i = 0; i < (1 << 8); ++i) {
                 ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32(i);
-                ggml_table_f32_f8_e4m3[i] = ggml_f8_e4m3_to_fp32(i);
             }
 
             const uint64_t t_end = ggml_time_us(); UNUSED(t_end);

ggml/src/ggml-cpu/llamafile/sgemm.cpp

@@ -69,7 +69,7 @@
 #define VECTOR_REGISTERS 16
 #endif
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 #define LMUL 4
 #endif
 
@@ -180,7 +180,7 @@ inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {
 }
 #endif
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 template <> inline vfloat32m1_t madd(vfloat32m1_t a, vfloat32m1_t b, vfloat32m1_t c) {
     return __riscv_vfmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1());
 }
@@ -277,7 +277,7 @@ inline float hsum(__m512 x) {
 }
 #endif // __AVX512F__
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 inline float hsum(vfloat32m1_t x) {
     return __riscv_vfmv_f_s_f32m1_f32(
         __riscv_vfredusum_vs_f32m1_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m1()));
@@ -458,7 +458,7 @@ template <> inline __m512bh load_partial(const ggml_bf16_t *p, int n) {
 #endif
 #endif
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 template <> inline vfloat32m1_t load(const float *p) {
     return __riscv_vle32_v_f32m1(p, __riscv_vsetvlmax_e32m1());
 }
@@ -503,7 +503,7 @@ template <> inline vbfloat16m4_t load(const ggml_bf16_t *p) {
 }
 #endif
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 template <typename T> T set_zero();
 
 template <> inline vfloat32m1_t set_zero() {
@@ -520,7 +520,7 @@ template <> inline vfloat32m8_t set_zero() {
 }
 #endif
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 template <typename T> size_t vlmax() {
     if constexpr (std::is_same_v<T, vfloat32m1_t>) { return  __riscv_vsetvlmax_e32m1(); }
     else if constexpr (std::is_same_v<T, vfloat32m2_t>) { return  __riscv_vsetvlmax_e32m2(); }
@@ -732,7 +732,7 @@ class tinyBLAS {
     const int64_t ldc;
 };
 
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 template <typename D, typename V, typename TA, typename TB, typename TC>
 class tinyBLAS_RVV {
   public:
@@ -3958,7 +3958,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
             params->ith, params->nth};
         tb.matmul(m, n);
         return true;
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
     #if LMUL == 1
         tinyBLAS_RVV<vfloat32m1_t, vfloat32m1_t, float, float, float> tb{ params,
             k, (const float *)A, lda,

ggml/src/ggml-cpu/ops.cpp

@@ -537,36 +537,26 @@ void ggml_compute_forward_dup(
     switch (src0->type) {
         case GGML_TYPE_F16:
             {
-                /**/ if (dst->type == GGML_TYPE_F16)     ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_BF16)    ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_F32)     ggml_compute_forward_dup_flt<ggml_fp16_t, float      >(params, dst);
-                else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp8_e4m3_t>(params, dst);
+                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_fp16_t, float      >(params, dst);
                 else ggml_compute_forward_dup_to_q<ggml_fp16_t>(params, dst);
             } break;
         case GGML_TYPE_BF16:
             {
-                /**/ if (dst->type == GGML_TYPE_F16)     ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_BF16)    ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_F32)     ggml_compute_forward_dup_flt<ggml_bf16_t, float      >(params, dst);
-                else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp8_e4m3_t>(params, dst);
+                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_bf16_t, float      >(params, dst);
                 else ggml_compute_forward_dup_to_q<ggml_bf16_t>(params, dst);
             } break;
         case GGML_TYPE_F32:
             {
-                /**/ if (dst->type == GGML_TYPE_F16)     ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_BF16)    ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_F32)     ggml_compute_forward_dup_flt<float, float      >(params, dst);
-                else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<float, ggml_fp8_e4m3_t>(params, dst);
-                else if (dst->type == GGML_TYPE_I32)     ggml_compute_forward_dup_flt<float, int32_t    >(params, dst);
+                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst);
+                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<float, float      >(params, dst);
+                else if (dst->type == GGML_TYPE_I32)  ggml_compute_forward_dup_flt<float, int32_t    >(params, dst);
                 else ggml_compute_forward_dup_to_q<float>(params, dst);
             } break;
-        case GGML_TYPE_F8_E4M3:
-            {
-                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, ggml_fp16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, ggml_bf16_t>(params, dst);
-                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, float      >(params, dst);
-                else GGML_ABORT("not implemented");
-            } break;
         case GGML_TYPE_I32:
             {
                 if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<int32_t, float>(params, dst);
@@ -615,9 +605,8 @@ static void ggml_compute_forward_add_q_f32(
     GGML_ASSERT(nb1 <= nb2);
     GGML_ASSERT(nb2 <= nb3);
 
-    GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3);
+    GGML_ASSERT(ggml_is_quantized(src0->type));
     GGML_ASSERT(src1->type == GGML_TYPE_F32);
-    GGML_ASSERT(ne00 % ggml_blck_size(type) == 0);
 
     // rows per thread
     const int dr = (nr + nth - 1)/nth;
@@ -647,6 +636,8 @@ static void ggml_compute_forward_add_q_f32(
         float * src1_row = (float *)((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13));
         void  * dst_row  = (void *) ((char *)  dst->data + ( i1*nb1  +  i2*nb2  +  i3*nb3));
 
+        assert(ne00 % 32 == 0);
+
         // unquantize row from src0 to temp buffer
         dequantize_row_q(src0_row, wdata, ne00);
         // add src1
@@ -682,7 +673,6 @@ void ggml_compute_forward_add(
         case GGML_TYPE_Q8_0:
         case GGML_TYPE_MXFP4:
         case GGML_TYPE_NVFP4:
-        case GGML_TYPE_F8_E4M3:
         case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
@@ -965,7 +955,7 @@ static void ggml_compute_forward_add1_q_f32(
     GGML_ASSERT(nb1 <= nb2);
     GGML_ASSERT(nb2 <= nb3);
 
-    GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3);
+    GGML_ASSERT(ggml_is_quantized(src0->type));
     GGML_ASSERT(dst->type == src0->type);
     GGML_ASSERT(src1->type == GGML_TYPE_F32);
 
@@ -1140,7 +1130,6 @@ void ggml_compute_forward_add1(
         case GGML_TYPE_Q8_1:
         case GGML_TYPE_MXFP4:
         case GGML_TYPE_NVFP4:
-        case GGML_TYPE_F8_E4M3:
         case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
@@ -1272,7 +1261,6 @@ void ggml_compute_forward_acc(
         case GGML_TYPE_Q8_1:
         case GGML_TYPE_MXFP4:
         case GGML_TYPE_NVFP4:
-        case GGML_TYPE_F8_E4M3:
         case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
@@ -4816,7 +4804,6 @@ void ggml_compute_forward_out_prod(
         case GGML_TYPE_Q8_0:
         case GGML_TYPE_MXFP4:
         case GGML_TYPE_NVFP4:
-        case GGML_TYPE_F8_E4M3:
         case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
@@ -5142,7 +5129,6 @@ void ggml_compute_forward_set(
         case GGML_TYPE_Q8_1:
         case GGML_TYPE_MXFP4:
         case GGML_TYPE_NVFP4:
-        case GGML_TYPE_F8_E4M3:
         case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
@@ -5368,7 +5354,6 @@ void ggml_compute_forward_get_rows(
         case GGML_TYPE_Q8_1:
         case GGML_TYPE_MXFP4:
         case GGML_TYPE_NVFP4:
-        case GGML_TYPE_F8_E4M3:
         case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
@@ -6129,7 +6114,6 @@ void ggml_compute_forward_clamp(
         case GGML_TYPE_Q8_1:
         case GGML_TYPE_MXFP4:
         case GGML_TYPE_NVFP4:
-        case GGML_TYPE_F8_E4M3:
         case GGML_TYPE_Q2_K:
         case GGML_TYPE_Q3_K:
         case GGML_TYPE_Q4_K:
@@ -8844,7 +8828,6 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
     const ggml_tensor * v     = dst->src[2];
     const ggml_tensor * mask  = dst->src[3];
     const ggml_tensor * sinks = dst->src[4];
-    const ggml_tensor * kv_rows = dst->src[5];
 
     GGML_TENSOR_LOCALS(int64_t, neq, q,   ne)
     GGML_TENSOR_LOCALS(size_t,  nbq, q,   nb)
@@ -8952,8 +8935,6 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
         const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));
         q_to_vec_dot(pq, Q_q, DK);
 
-        const int32_t * rows = kv_rows ? (const int32_t *) ((const char *) kv_rows->data + iq3*kv_rows->nb[1]) : nullptr;
-
         // online softmax / attention
         // loop over n_kv and n_head_kv
         // ref: https://arxiv.org/pdf/2112.05682.pdf
@@ -8964,14 +8945,9 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
                 continue;
             }
 
-            const int64_t ik1 = rows ? rows[ic] : ic;
-            if (ik1 < 0) {
-                continue;
-            }
-
             float s; // KQ value
 
-            const char * k_data = (const char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3);
+            const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
             kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);
 
             s = s*scale; // scale KQ value
@@ -8987,7 +8963,7 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
             float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value
             float vs = 1.0f; // post-softmax KQ value, expf(s - M)
 
-            const char * v_data = ((const char *) v->data + (ik1*nbv1 + iv2*nbv2 + iv3*nbv3));
+            const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
 
             if (v->type == GGML_TYPE_F16) {
                 if (s > M) {
@@ -9154,10 +9130,6 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
     static constexpr int Q_TILE_SZ  = ggml_fa_tile_config::Q;
     static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV;
 
-    // with kv_rows the loop runs over the mask columns and reads K/V rows through kv_rows
-    const ggml_tensor * kv_rows = dst->src[5];
-    const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : nek1;
-
     int ir = ir0;
     while (ir < ir1) {
         // q indices for the start of this tile
@@ -9165,10 +9137,6 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
         const int iq2 = (ir - iq3*neq2*neq1)/neq1;
         const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
 
-        // padding rows are masked, read row 0 for them
-        const int32_t * rows = kv_rows ? (const int32_t *) ((const char *) kv_rows->data + iq3*kv_rows->nb[1]) : nullptr;
-        auto kv_row = [rows](int64_t i) -> int64_t { return rows ? std::max<int32_t>(rows[i], 0) : i; };
-
         // Number of valid rows in this tile:
         // - limited by tile size (Q_TILE_SZ)
         // - limited by chunk boundary (ir1 - ir)
@@ -9228,8 +9196,8 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
         memset(K_f32, 0, DK * KV_TILE_SZ * sizeof(float));
         memset(V32,   0, KV_TILE_SZ * DV * sizeof(float));
 
-        for (int64_t ic = 0; ic < n_kv_cols; ic += KV_TILE_SZ) {
-            const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, n_kv_cols - ic);
+        for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) {
+            const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, nek1 - ic);
 
             // skip the tile entirely if all the masks are -inf
             if (mask) {
@@ -9256,7 +9224,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
             // Pack K tile transposed: K_f32[dk][kv] so KV_TILE is contiguous (SIMD dim)
             // Zero-pad the last tile so the GEMM always operates on KV_TILE_SZ columns
             for (int tk = 0; tk < kv_tile; tk++) {
-                const char * k_data = (const char *)k->data + kv_row(ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
+                const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
                 if (kv_type == GGML_TYPE_F16) {
                     const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
                     for (int64_t dk = 0; dk < DK; dk++) {
@@ -9321,7 +9289,7 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
             // V accumulation: VKQ32 += softmax(KQ) * V
             // Pack V tile to contiguous F32, zero-padded
             for (int tk = 0; tk < kv_tile; tk++) {
-                const char * v_data = (const char *)v->data + kv_row(ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
+                const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
                 if (kv_type == GGML_TYPE_F16) {
                     ggml_cpu_fp16_to_fp32((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);
                 } else {
@@ -9493,12 +9461,8 @@ static void ggml_compute_forward_flash_attn_ext_f16(
     // When use_ref is set, force the vec-only reference implementation (no tiling, no KV-chunking)
     const bool use_ref = params->use_ref;
 
-    // with kv_rows the loops run over the mask columns, the split KV path does not support it
-    const ggml_tensor * kv_rows = dst->src[5];
-    const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : nek1;
-
     const bool kv_is_f32_or_f16 = (k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_F16);
-    const bool use_split_kv_path = !use_ref && !kv_rows && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512;
+    const bool use_split_kv_path = !use_ref && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512;
 
     if (use_split_kv_path) {
         const int64_t chunk_size = (nek1 + nth - 1) / nth;
@@ -9577,7 +9541,7 @@ static void ggml_compute_forward_flash_attn_ext_f16(
             if (use_tiled) {
                 ggml_compute_forward_flash_attn_ext_tiled(params, dst, ir0, ir1);
             } else {
-                ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, n_kv_cols, nullptr, 0);
+                ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, nek1, nullptr, 0);
             }
 
             current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
@@ -10114,7 +10078,7 @@ static void ggml_compute_forward_ssm_scan_f32(
                         }
 
                         sumf = GGML_F32xt_REDUCE_ONE(sum);
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
                         // todo: RVV implementation
                         const int np = 0;
     #else
@@ -11778,7 +11742,7 @@ static void ggml_compute_forward_rwkv_wkv7_f32(
     int64_t h_stride_2d = head_size * head_size;
 
     #if defined(GGML_SIMD)
-        #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v)
+        #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v_intrinsic)
             // scalar Route to scalar implementation       //TODO: Write SVE code and RVV code
             for (int64_t t = 0; t < T; t++) {
                 int64_t t_offset = t * t_stride;

ggml/src/ggml-cpu/quants.c

@@ -3,7 +3,6 @@
 
 #include "ggml-cpu-impl.h"
 #include "simd-mappings.h"
-#include "vec.h"
 #include "ggml-quants.h"
 #include "quants.h"
 
@@ -63,10 +62,6 @@ void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i
     quantize_row_nvfp4_ref(x, y, k);
 }
 
-void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
-    quantize_row_f8_e4m3_ref(x, y, k);
-}
-
 //
 // 2-6 bit quantization in super-blocks
 //
@@ -367,49 +362,6 @@ void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs,
     *s = sumf;
 }
 
-void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
-    assert(nrc == 1);
-    UNUSED(nrc);
-    UNUSED(bx);
-    UNUSED(by);
-    UNUSED(bs);
-
-    const ggml_fp8_e4m3_t * GGML_RESTRICT x = vx;
-    const float * GGML_RESTRICT y = vy;
-
-    float sumf = 0.0f;
-    int pos = 0;
-#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)
-    GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };
-    const int np = n - n % GGML_F32_STEP;
-    for (; pos < np; pos += GGML_F32_STEP) {
-        for (int j = 0; j < GGML_F32_ARR; ++j) {
-            const int offset = pos + j*GGML_F32_EPR;
-            GGML_F32_VEC ax;
-#if defined(__AVX512F__) && defined(__AVX512BW__)
-            const __m128i codes = _mm_loadu_si128((const __m128i *) (x + offset));
-            ax = _mm512_i32gather_ps(_mm512_cvtepu8_epi32(codes), ggml_table_f32_f8_e4m3, sizeof(float));
-#elif defined(__AVX2__) && !defined(__AVX512F__)
-            const __m128i codes = _mm_loadl_epi64((const __m128i *) (x + offset));
-            ax = _mm256_i32gather_ps(ggml_table_f32_f8_e4m3, _mm256_cvtepu8_epi32(codes), sizeof(float));
-#else
-            float values[GGML_F32_EPR];
-            for (int k = 0; k < GGML_F32_EPR; ++k) {
-                values[k] = ggml_table_f32_f8_e4m3[x[offset + k].bits];
-            }
-            ax = GGML_F32_VEC_LOAD(values);
-#endif
-            sum[j] = GGML_F32_VEC_FMA(sum[j], ax, GGML_F32_VEC_LOAD(y + offset));
-        }
-    }
-    GGML_F32_VEC_REDUCE(sumf, sum);
-#endif
-    for (; pos < n; ++pos) {
-        sumf += ggml_table_f32_f8_e4m3[x[pos].bits] * y[pos];
-    }
-    *s = sumf;
-}
-
 void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
     const int qk = QK8_0;
     const int nb = n / qk;

ggml/src/ggml-cpu/quants.h

@@ -23,7 +23,6 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in
 
 void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
 void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
-void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
 
 void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
 void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
@@ -41,7 +40,6 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y,
 // Dot product
 void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
 void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
-void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
 void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
 void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
 void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);

ggml/src/ggml-cpu/simd-gemm.h

@@ -5,7 +5,7 @@
 #include "simd-mappings.h"
 
 // TODO: add support for sizeless vector types
-#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v)
+#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)
 
 // TODO: untested on avx512
 // These are in units of GGML_F32_EPR
@@ -149,7 +149,7 @@ static void simd_gemm(
         C += N;
     }
 }
-#elif defined(GGML_SIMD) && defined(__riscv_v)
+#elif defined(GGML_SIMD) && defined(__riscv_v_intrinsic)
 // RM accumulators + 1 B vector = RM + 1 <= 8  =>  RM <= 7
 // Microkernel: C[RM x vl] += A[RM x K] * B[K x N]
 template <int RM>

ggml/src/ggml-cpu/simd-mappings.h

@@ -14,9 +14,7 @@
 #include <arm_neon.h>
 #endif
 
-// __riscv_v_intrinsic reports compiler API support and can be defined without
-// enabling V; use the ISA feature macro to select RVV code.
-#if defined(__riscv_v)
+#if defined(__riscv_v_intrinsic)
 #include <riscv_vector.h>
 #endif
 
@@ -129,8 +127,6 @@ extern float ggml_table_f32_e8m0_half[1 << 8];
 // defined in ggml-cpu.c, initialized in ggml_cpu_init()
 extern float ggml_table_f32_ue4m3[1 << 8];
 
-extern float ggml_table_f32_f8_e4m3[1 << 8];
-
 // Use lookup table for E8M0 on x86 (faster than bit manipulation)
 #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
 #define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)]
@@ -1308,7 +1304,7 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) {
 #define GGML_BF16_FMA_HI(acc, x, y) \
     (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))
 
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
 
 // compatible with vlen >= 128
 

ggml/src/ggml-cpu/spacemit/ime.cpp

@@ -1046,10 +1046,6 @@ class tensor_traits_common : public tensor_traits_base {
                         return true;
                 }
             case GGML_OP_FLASH_ATTN_EXT:
-                // K/V row lists are handled by the generic CPU path
-                if (op->src[5] != nullptr) {
-                    return false;
-                }
                 forward_flash_attn_ext_f16(params, op);
                 return true;
             case GGML_OP_CONT:
@@ -1493,9 +1489,6 @@ static const ggml_backend_buffer_i ggml_backend_riscv64_spacemit_buffer_i = {
     /* .cpy_tensor      = */ nullptr,
     /* .clear           = */ ggml_backend_riscv64_spacemit_buffer_clear,
     /* .reset           = */ nullptr,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 static const char * ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name(ggml_backend_buffer_type_t buft) {

ggml/src/ggml-cpu/vec.cpp

@@ -84,7 +84,7 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G
         }
         // reduce sum1,sum2 to sum1
         GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8);
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
         int vl = __riscv_vsetvlmax_e32m8();
         vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);
         vfloat32m8_t vsum;
@@ -195,7 +195,7 @@ void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t *
     sumf += (ggml_float)_mm_cvtss_f32(g);
 
 #undef LOAD
-#elif defined(__riscv_v) && defined(__riscv_zvfbfwma)
+#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma)
     size_t vl = __riscv_vsetvlmax_e32m4();
 
     // initialize accumulators to all zeroes
@@ -318,7 +318,7 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G
         sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum3_hi);
 
         sumf = ggml_sve_sum_f32x2(sum1_lo, sum1_hi);
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
         #if defined(__riscv_zvfh)
             int vl = __riscv_vsetvlmax_e32m2();
             vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);
@@ -405,7 +405,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {
     for (; i + 3 < n; i += 4) {
         vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i);
     }
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
     for (int vl; i < n; i += vl) {
         vl = __riscv_vsetvl_e32m2(n - i);
         vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);
@@ -446,7 +446,7 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float *
     for (; i + 3 < n; i += 4) {
         vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i);
     }
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
     for (int vl; i < n; i += vl) {
         vl = __riscv_vsetvl_e32m2(n - i);
         vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);
@@ -602,7 +602,7 @@ ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const floa
         val = vec_mul(val, val);
         sum += (ggml_float)vec_hsum_f32x4(val);
     }
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
     vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);
     for (int vl; i < n; i += vl) {
         vl = __riscv_vsetvl_e32m2(n - i);
@@ -681,7 +681,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float
         vec_xst(v_val, 0, y + i);
         sum += (ggml_float)vec_hsum_f32x4(v_val);
     }
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
     vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);
     for (int avl; i < n; i += avl) {
         avl = __riscv_vsetvl_e32m2(n - i);

ggml/src/ggml-cpu/vec.h

@@ -206,7 +206,7 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG
         sumf[0] = ggml_sve_sum_f32x2(sum_0_lo, sum_0_hi);
         sumf[1] = ggml_sve_sum_f32x2(sum_1_lo, sum_1_hi);
         np = n;
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
         #if defined(__riscv_zvfh)
             size_t vl = __riscv_vsetvlmax_e32m4();
 
@@ -396,7 +396,7 @@ inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const
 
             svst1_f32(pg, y + np2, ay1);
         }
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
         for (int i = 0, avl; i < n; i += avl) {
             avl = __riscv_vsetvl_e32m8(n - i);
             vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);
@@ -513,7 +513,7 @@ inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y,
         svst1_f16(pg, (__fp16 *)(y + np2), hy);
     }
     np = n;
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic) // implies __riscv_v_intrinsic
     #if defined (__riscv_zvfh)
         const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);
         const _Float16 scale = *(const _Float16*)(&s);
@@ -599,7 +599,7 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int
                 y[i] += x[k][i]*v[k][0];
             }
         }
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
         for (int i = 0, avl; i < n; i += avl) {
             avl = __riscv_vsetvl_e32m8(n - i);
             vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);
@@ -660,7 +660,7 @@ inline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, co
         for (int i = 0; i < n; ++i) {
             y[i] = x[i]*s + b;
         }
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
         for (int i = 0, avl; i < n; i += avl) {
             avl = __riscv_vsetvl_e32m8(n - i);
             vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);
@@ -729,7 +729,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float   v) {
             ay1 = svmul_f32_m(pg, ay1, vx);
             svst1_f32(pg, y + i, ay1);
         }
-    #elif defined(__riscv_v)
+    #elif defined(__riscv_v_intrinsic)
         for (int i = 0, avl; i < n; i += avl) {
             avl = __riscv_vsetvl_e32m8(n - i);
             vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);
@@ -793,7 +793,7 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float
         svst1_f16(pg, (__fp16 *)(y + np), out);
     }
     np = n;
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
     #if defined(__riscv_zvfh)
         const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);
         const _Float16 scale = *(const _Float16*)(&s);
@@ -1411,7 +1411,7 @@ inline static float32x4_t ggml_v_gelu_quick(float32x4_t x) {
     return vec_div(x, vec_add(one, exp_cx));
 }
 
-#elif defined(__riscv_v)
+#elif defined(__riscv_v_intrinsic)
 
 // adapted from arm limited optimized routine
 // the maximum error is 1.45358 plus 0.5 ulps
@@ -1469,7 +1469,7 @@ inline static vfloat32m2_t ggml_v_gelu_quick_m2(vfloat32m2_t x, int vl) {
     return __riscv_vfdiv_vv_f32m2(x, __riscv_vfadd_vf_f32m2(exp_cx, 1.0f, vl), vl);
 }
 
-#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v
+#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v_intrinsic
 
 inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
     for (int i = 0; i < n; ++i) {

ggml/src/ggml-cuda/batch-get.cu

@@ -1,198 +0,0 @@
-#include "batch-get.cuh"
-#include "common.cuh"
-#include "ggml-cpp.h"
-
-#include <climits>
-
-#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
-struct ggml_cuda_batch_get_descriptor {
-    const unsigned char * src;
-    unsigned char * dst;
-    size_t size;
-};
-
-static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) {
-    // Blocks along y cooperate on each descriptor.
-    const size_t index = blockIdx.x;
-    if (index >= count) {
-        return;
-    }
-    const size_t first = (size_t) blockIdx.y * blockDim.x + threadIdx.x;
-    const size_t stride = (size_t) gridDim.y * blockDim.x;
-    const auto copy = copies[index];
-    // Use 16-, 8-, 4-, or 2-byte copies when both pointers meet that alignment; threads copy separate chunks.
-    const uintptr_t alignment = (uintptr_t) copy.src | (uintptr_t) copy.dst;
-    size_t tail = 0;
-    if (alignment % 16 == 0) {
-        for (size_t i = first; i < copy.size / 16; i += stride) {
-            ggml_cuda_memcpy_1<16>(copy.dst + 16*i, copy.src + 16*i);
-        }
-        tail = copy.size / 16 * 16;
-    } else if (alignment % 8 == 0) {
-        for (size_t i = first; i < copy.size / 8; i += stride) {
-            ggml_cuda_memcpy_1<8>(copy.dst + 8*i, copy.src + 8*i);
-        }
-        tail = copy.size / 8 * 8;
-    } else if (alignment % 4 == 0) {
-        for (size_t i = first; i < copy.size / 4; i += stride) {
-            ggml_cuda_memcpy_1<4>(copy.dst + 4*i, copy.src + 4*i);
-        }
-        tail = copy.size / 4 * 4;
-    } else if (alignment % 2 == 0) {
-        for (size_t i = first; i < copy.size / 2; i += stride) {
-            ggml_cuda_memcpy_1<2>(copy.dst + 2*i, copy.src + 2*i);
-        }
-        tail = copy.size / 2 * 2;
-    }
-    // Copy trailing bytes, or the entire range when no wider alignment is available.
-    for (size_t i = tail + first; i < copy.size; i += stride) {
-        ggml_cuda_memcpy_1<1>(copy.dst + i, copy.src + i);
-    }
-}
-
-struct ggml_cuda_batch_get_mapping {
-    void * host = nullptr;
-    unsigned char * device = nullptr;
-
-    ggml_cuda_batch_get_mapping(void * host, size_t size) {
-        const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable);
-        if (err != cudaSuccess) {
-            (void) cudaGetLastError();
-            GGML_LOG_DEBUG("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__,
-                           size / 1024.0 / 1024.0, cudaGetErrorString(err));
-            return;
-        }
-        this->host = host;
-        if (cudaHostGetDevicePointer((void **) &device, host, 0) != cudaSuccess) {
-            (void) cudaGetLastError();
-            device = nullptr;
-            return;
-        }
-    }
-
-    ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete;
-    ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete;
-
-    ~ggml_cuda_batch_get_mapping() {
-        if (host) {
-            // Release only the registration acquired by this object.
-            CUDA_CHECK(cudaHostUnregister(host));
-        }
-    }
-};
-#endif
-
-bool ggml_cuda_get_tensor_n(int device,
-        const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
-        const size_t * sizes, size_t n_tensors) {
-#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
-    GGML_UNUSED(device);
-    GGML_UNUSED(tensors);
-    GGML_UNUSED(data);
-    GGML_UNUSED(offsets);
-    GGML_UNUSED(sizes);
-    GGML_UNUSED(n_tensors);
-    return false;
-#else
-    // Provisional dispatch values based on experiments.
-    constexpr size_t min_batch_reads = 128;
-    constexpr size_t max_gather_read_bytes = 128 * 1024;
-    // Collect non-empty CPU destination ranges.
-    std::vector<std::pair<uintptr_t, uintptr_t>> ranges;
-    ranges.reserve(n_tensors);
-    size_t payload_bytes = 0;
-    // Number of non-empty reads whose size is at most max_gather_read_bytes.
-    size_t n_reads_small = 0;
-    for (size_t i = 0; i < n_tensors; ++i) {
-        if (sizes[i] == 0) {
-            continue;
-        }
-        const uintptr_t address = (uintptr_t) data[i];
-        GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]);
-        GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]);
-        payload_bytes += sizes[i];
-        n_reads_small += sizes[i] <= max_gather_read_bytes;
-        ranges.emplace_back(address, address + sizes[i]);
-    }
-    if (ranges.empty()) {
-        return true;
-    }
-    // Total number of non-empty reads.
-    const size_t n_reads = ranges.size();
-    if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) {
-        return false;
-    }
-    if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) {
-        return false;
-    }
-    std::sort(ranges.begin(), ranges.end());
-    for (size_t i = 1; i < ranges.size(); ++i) {
-        // Preserve the ordered copy path when CPU destinations overlap.
-        if (ranges[i].first < ranges[i - 1].second) {
-            return false;
-        }
-    }
-    // Find the CPU span to register, including gaps between destinations.
-    const uintptr_t begin = ranges.front().first;
-    const uintptr_t end = ranges.back().second;
-    if (end - begin - payload_bytes > payload_bytes) {
-        return false;
-    }
-    // Register and map the existing CPU destination so the kernel can write to it directly.
-    ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin);
-    if (!mapping.host) {
-        return false;
-    }
-    // Reads selected for gather; zero means all reads use DMA.
-    size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
-    // Keep descriptor storage alive until the final stream synchronization.
-    constexpr size_t max_batch_descriptors = 128 * 1024;
-    const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor);
-    ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
-    if (!descriptors) {
-        n_reads_gather = 0;
-    }
-    const auto host_type = ggml_backend_cuda_host_buffer_type();
-    ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
-    if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) {
-        n_reads_gather = 0;
-    }
-    if (n_reads_gather) {
-        auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
-        auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
-        const int threads = 256;
-        const size_t nsm = ggml_cuda_info().devices[device].nsm;
-        size_t i = 0;
-        for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) {
-            const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors);
-            size_t count = 0;
-            for (; i < n_tensors && count < n_batch; ++i) {
-                if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) {
-                    continue;
-                }
-                copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i],
-                        mapping.device + ((uintptr_t) data[i] - begin), sizes[i]};
-            }
-            const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch));
-            const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy);
-            // Stream ordering keeps each kernel ahead of the next descriptor overwrite.
-            CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread));
-            ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
-            CUDA_CHECK(cudaGetLastError());
-            if (first + n_batch < n_reads_gather) {
-                // Wait before refilling the pinned host descriptor buffer.
-                CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
-            }
-        }
-    }
-    // Keep the destination pinned for DMA too; overlapping destinations already declined above.
-    for (size_t i = 0; i < n_tensors; ++i) {
-        if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) {
-            CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread));
-        }
-    }
-    // Wait for GPU writes before releasing the CPU mapping and descriptor buffers.
-    CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
-    return true;
-#endif
-}

ggml/src/ggml-cuda/batch-get.cuh

@@ -1,7 +0,0 @@
-#pragma once
-
-#include "ggml-backend.h"
-
-bool ggml_cuda_get_tensor_n(int device,
-        const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
-        const size_t * sizes, size_t n_tensors);

ggml/src/ggml-cuda/common.cuh

@@ -82,7 +82,6 @@
 #define GGML_CUDA_CC_RDNA2      (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a
 #define GGML_CUDA_CC_RDNA3      (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA
 #define GGML_CUDA_CC_RDNA3_5    (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops.
-#define GGML_CUDA_CC_RDNA4_M    (GGML_CUDA_CC_OFFSET_AMD + 0x1170) // gfx117X
 #define GGML_CUDA_CC_RDNA4      (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000
 
 #define GGML_CUDA_CC_IS_AMD(cc)     (cc >= GGML_CUDA_CC_OFFSET_AMD)
@@ -90,9 +89,8 @@
 #define GGML_CUDA_CC_IS_RDNA1(cc)   (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)
 #define GGML_CUDA_CC_IS_RDNA2(cc)   (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)
 #define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5)
-#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4_M)
+#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4)
 #define GGML_CUDA_CC_IS_RDNA3(cc)   (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc))
-#define GGML_CUDA_CC_IS_RDNA4_M(cc) (cc >= GGML_CUDA_CC_RDNA4_M && cc < GGML_CUDA_CC_RDNA4)
 #define GGML_CUDA_CC_IS_RDNA4(cc)   (cc >= GGML_CUDA_CC_RDNA4)
 #define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C)
 #define GGML_CUDA_CC_IS_GCN(cc)     ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc))
@@ -279,17 +277,9 @@ static const char * cu_get_error_str(CUresult err) {
 #define AMD_MFMA_AVAILABLE
 #endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)
 
-#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA4_M))
-#define AMD_WMMA_128B_AVAILABLE
-#endif // defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA4_M))
-
-#if defined(GGML_USE_HIP) && defined(RDNA3)
-#define AMD_WMMA_256B_AVAILABLE
-#endif // defined(GGML_USE_HIP) && defined(RDNA3)
-
-#if defined(AMD_WMMA_128B_AVAILABLE) || defined(AMD_WMMA_256B_AVAILABLE)
+#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3))
 #define AMD_WMMA_AVAILABLE
-#endif // defined(AMD_WMMA_128B_AVAILABLE) || defined(AMD_WMMA_256B_AVAILABLE)
+#endif // defined(GGML_USE_HIP) && defined(RDNA4)
 
 // The Volta instructions are in principle available on Turing or newer but they are effectively unusable:
 #if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA
@@ -336,7 +326,7 @@ static bool fast_fp16_hardware_available(const int cc) {
 // To be used for feature selection of external libraries, e.g. cuBLAS.
 static bool fp16_mma_hardware_available(const int cc) {
     return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) ||
-        GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||
+        GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||
         (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);
 }
 
@@ -365,7 +355,7 @@ static bool amd_mfma_available(const int cc) {
 }
 
 static bool amd_wmma_available(const int cc) {
-    return GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc);
+    return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc));
 }
 
 static bool volta_mma_available(const int cc) {
@@ -723,7 +713,7 @@ static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, i
 #if defined(GGML_USE_HIP)
 #if defined(CDNA) || defined(RDNA2) || defined(__gfx906__)
     c = __builtin_amdgcn_sdot4(a, b, c, false);
-#elif defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4)
+#elif defined(RDNA3) || defined(RDNA4)
     c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);
 #elif defined(RDNA1) || defined(__gfx900__)
     int tmp1;
@@ -768,9 +758,9 @@ static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v
     acc += v.y*u.y;
 }
 
-#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
+#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
 #define V_DOT2_F32_F16_AVAILABLE
-#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
+#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
 
 static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) {
 #ifdef V_DOT2_F32_F16_AVAILABLE
@@ -1469,21 +1459,16 @@ struct ggml_backend_cuda_context {
     // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe)
     std::unordered_map<const void *, std::unique_ptr<ggml_cuda_graph>> cuda_graphs;
 
-    int64_t next_graph_eviction_time = 0;
-    int64_t last_graph_use_time = 0;
+    int64_t last_graph_eviction_sweep = 0;
 
-    ggml_cuda_graph * cuda_graph(const void * first_node_ptr, int64_t time_now = ggml_time_us()) {
-        // Give all cached graphs 10s to be reused after an idle period, without updating each graph.
-        if (!cuda_graphs.empty() && time_now - last_graph_use_time >= 10'000'000) {
-            next_graph_eviction_time = time_now + 10'000'000;
-        }
-        last_graph_use_time = time_now;
+    ggml_cuda_graph * cuda_graph(const void * first_node_ptr) {
+        const int64_t time_now = ggml_time_us();
 
-        // Sweep every 5s during active use, evicting graphs unused for at least 10s.
-        if (time_now >= next_graph_eviction_time) {
-            next_graph_eviction_time = time_now + 5'000'000;
+        // sweep every 5s, evicting cuda graphs unused for >=10s
+        if (time_now - last_graph_eviction_sweep >= 5'000'000) {
+            last_graph_eviction_sweep = time_now;
             for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) {
-                if (it->first != first_node_ptr && time_now - it->second->last_used_time >= 10'000'000) {
+                if (time_now - it->second->last_used_time >= 10'000'000) {
                     it = cuda_graphs.erase(it);
                 } else {
                     ++it;

ggml/src/ggml-cuda/conv2d.cu

@@ -180,6 +180,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
     const int                  end            = int(int64_t(tiles) * (split + 1) / split_k) * BK;
     for (int k0 = begin; k0 < end; k0 += BK) {
         if (k_total % 8 == 0 && uintptr_t(weight) % 16 == 0) {
+#pragma unroll
             for (int i = tid; i < BM * BK / 8; i += nthreads) {
                 const int  row = i / (BK / 8), col = 8 * (i % (BK / 8));
                 const int4 v                 = m0 + row < oc && k0 + col < k_total ?
@@ -188,6 +189,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 *(int4 *) &a_s[row][col / 2] = v;
             }
         } else {
+#pragma unroll
             for (int i = tid; i < BM * BK / 2; i += nthreads) {
                 const int row = i / (BK / 2), col = 2 * (i % (BK / 2));
                 half      lo = __float2half(0.0f), hi = lo;
@@ -200,6 +202,7 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 a_s[row][col / 2] = __halves2half2(lo, hi);
             }
         }
+#pragma unroll
         for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
             const int ki = k0 + k;
             const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;

ggml/src/ggml-cuda/fattn-common.cuh

@@ -25,7 +25,6 @@ typedef void (* fattn_kernel_t)(
         const char * __restrict__ mask,
         const char * __restrict__ sinks,
         const int  * __restrict__ KV_max,
-        const int  * __restrict__ KV_rows,
         float      * __restrict__ dst,
         float2     * __restrict__ dst_meta,
         const float scale,
@@ -992,13 +991,8 @@ void launch_fattn(
 
     const bool V_is_K_view = V->view_src && (V->view_src == K || (V->view_src == K->view_src && V->view_offs == K->view_offs));
 
-    const ggml_tensor * mask    = dst->src[3];
-    const ggml_tensor * sinks   = dst->src[4];
-    const ggml_tensor * kv_rows = dst->src[5];
-
-    // with kv_rows the kernel iterates over the mask columns and reads K/V rows through kv_rows
-    const int64_t n_kv_cols = kv_rows ? kv_rows->ne[0] : K->ne[1];
-    GGML_ASSERT(!kv_rows || !use_sparse);
+    const ggml_tensor * mask  = dst->src[3];
+    const ggml_tensor * sinks = dst->src[4];
 
     ggml_tensor * KQV = dst;
 
@@ -1119,8 +1113,8 @@ void launch_fattn(
 
     // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.
     // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
-    //     multiple sequences of possibly different lengths, or kv_rows lists that end with padding.
-    const bool scan_mask = !use_sparse && mask && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
+    //     multiple sequences of possibly different lengths.
+    const bool scan_mask = !use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1);
     if (scan_mask) {
         const int64_t s31 = mask->nb[1] / sizeof(half2);
         const int64_t s33 = mask->nb[3] / sizeof(half2);
@@ -1129,7 +1123,7 @@ void launch_fattn(
         const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1);
 
         const int ne_KV_max = blocks_num_KV_max.x*blocks_num_KV_max.y;
-        const int iter_k = n_kv_cols / FATTN_KQ_STRIDE;
+        const int iter_k = K->ne[1] / FATTN_KQ_STRIDE;
 
         KV_max.alloc(ne_KV_max);
         ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(blocks_num_KV_max, block_dim_KV_max, 0, main_stream);
@@ -1138,19 +1132,13 @@ void launch_fattn(
         CUDA_CHECK(cudaGetLastError());
     }
 
-    // K/V are shared by all slices when read through kv_rows
-    if (kv_rows) {
-        nb13 = 0;
-        nb23 = 0;
-    }
-
     const dim3 block_dim(warp_size, nwarps, 1);
     int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy.
     CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared));
     GGML_ASSERT(max_blocks_per_sm > 0);
     int parallel_blocks = max_blocks_per_sm;
 
-    const int64_t n_kv = use_sparse ? n_kv_max : n_kv_cols;
+    const int64_t n_kv = use_sparse ? n_kv_max : K->ne[1];
     const int ntiles_KV = (n_kv + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.
 
     dim3 blocks_num;
@@ -1267,7 +1255,6 @@ void launch_fattn(
         mask ? ((const char *) mask->data) : nullptr,
         sinks ? ((const char *) sinks->data) : nullptr,
         KV_max.ptr,
-        kv_rows ? (const int *) kv_rows->data : nullptr,
         !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr,
         scale, max_bias, m0, m1, n_head_log2, logit_softcap,
         Q->ne[0], ne01,     Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3],

ggml/src/ggml-cuda/fattn-mma-f16.cuh

@@ -420,9 +420,6 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
                     // padded slots gather row 0, the -inf mask removes their contribution
                     const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : 0;
                     i_KV = index >= 0 ? index : 0;
-                } else if (indices) {
-                    // K/V rows given per mask column, negative rows are padding
-                    i_KV = max(indices[k_VKQ_0 + i], 0);
                 } else {
                     i_KV = k_VKQ_0 + i;
                 }
@@ -470,11 +467,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(
                     if constexpr (use_sparse) {
                         const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1;
                         src = index >= 0 ? KV + int64_t(index)*stride_KV + k*h2_per_chunk : zero;
-                    } else if (!oob_check || i < i_sup) {
-                        const int64_t i_KV = indices ? max(indices[k_VKQ_0 + i], 0) : k_VKQ_0 + i;
-                        src = KV + i_KV*stride_KV + k*h2_per_chunk;
                     } else {
-                        src = zero;
+                        src = !oob_check || i < i_sup ? KV + int64_t(k_VKQ_0 + i)*stride_KV + k*h2_per_chunk : zero;
                     }
                     ggml_cuda_memcpy_1<16>(swizzle<stride_tile>(tile_KV, i*stride_tile + k*h2_per_chunk, i), src);
                 }
@@ -640,7 +634,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
         cp_async_wait_all();
         __syncthreads();
         flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
-            (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, indices);
+            (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, nullptr);
     } else {
         // the sparse mask values are gathered per element, always load them synchronously
         constexpr bool use_cp_async = nstages == 1 && !use_sparse;
@@ -997,7 +991,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
                     (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
             }
             flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
-                (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices);
+                (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, nullptr);
         }
     }
 
@@ -1261,7 +1255,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
     T_B_KQ    Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)];
 #if defined(TURING_MMA_AVAILABLE)
     T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)];
-#elif defined(AMD_WMMA_256B_AVAILABLE)
+#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
     T_C_VKQ VKQ_C[DV % 32 != 0       ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)];
 #elif defined(AMD_MFMA_AVAILABLE)
     T_C_VKQ VKQ_C[                                     DV/T_C_VKQ::J];
@@ -1349,7 +1343,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
                 (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
         }
         flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
-            (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, indices);
+            (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr);
     }
 
     // kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
@@ -1808,7 +1802,6 @@ static __global__ void flash_attn_ext_f16(
         const char * mask_ptr,
         const char * sinks_ptr,
         const int  * KV_max_ptr,
-        const int  * KV_rows_ptr,
         float      * dst_ptr,
         float2     * dst_meta_ptr,
         const float scale,
@@ -1873,7 +1866,7 @@ static __global__ void flash_attn_ext_f16(
 #endif // defined(AMD_WMMA_AVAILABLE)
 
 #if defined(AMD_MFMA_AVAILABLE)
-    if (ncols1*ncols2 < 16 || (DKQ > 256 && ncols1*ncols2 < 32 && !use_sparse)) {
+    if (ncols1*ncols2 < 16 || (DKQ > 256 && ncols1*ncols2 < 32)) {
         NO_DEVICE_CODE;
         return;
     }
@@ -1931,8 +1924,7 @@ static __global__ void flash_attn_ext_f16(
 
         const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
         const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;
-        const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 :
-                                  KV_rows_ptr ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr;
+        const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr;
 
         const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
 
@@ -1981,8 +1973,7 @@ static __global__ void flash_attn_ext_f16(
 
     const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
     const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;
-    const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 :
-                              KV_rows_ptr ? KV_rows_ptr + int64_t(sequence)*ne11 : nullptr;
+    const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*iter_j + jt)*ne11 : nullptr;
 
     const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
 
@@ -1998,7 +1989,7 @@ static __global__ void flash_attn_ext_f16(
         (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
          ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
 #else
-    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
+    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,
         max_bias, m0, m1, n_head_log2, logit_softcap,
         ne00, ne01, ne02, ne03,
               nb01, nb02, nb03,
@@ -2063,7 +2054,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
     bool use_sparse = false;
     if (logit_softcap == 0.0f) {
         constexpr bool use_logit_softcap = false;
-#if !defined(GGML_USE_MUSA)
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
         if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2)) {
             if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, ncols1, ncols2)) {
                 constexpr bool use_sparse_kernel = true;
@@ -2086,7 +2077,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
                 }
             }
         } else
-#endif // !defined(GGML_USE_MUSA)
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
         {
             constexpr bool use_sparse_kernel = false;
             fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;

ggml/src/ggml-cuda/fattn-tile.cuh

@@ -797,7 +797,6 @@ static __global__ void flash_attn_tile(
         const char * mask_ptr,
         const char * sinks_ptr,
         const int  * KV_max_ptr,
-        const int  * KV_rows_ptr,
         float      * dst_ptr,
         float2     * dst_meta_ptr,
         const float scale,
@@ -822,7 +821,6 @@ static __global__ void flash_attn_tile(
     const int  * GGML_CUDA_RESTRICT KV_max   = KV_max_ptr;
     float      * GGML_CUDA_RESTRICT dst      = dst_ptr;
     float2     * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
-    GGML_UNUSED(KV_rows_ptr);
 
     // Skip unused kernel variants for faster compilation:
 
@@ -1134,7 +1132,7 @@ static __global__ void flash_attn_tile(
         }
     }
 #else
-    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
+    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,
         max_bias, m0, m1, n_head_log2, logit_softcap,
         ne00, ne01, ne02, ne03,
               nb01, nb02, nb03,

ggml/src/ggml-cuda/fattn-vec.cuh

@@ -25,7 +25,6 @@ static __global__ void flash_attn_ext_vec(
         const char * mask_ptr,
         const char * sinks_ptr,
         const int  * KV_max_ptr,
-        const int  * KV_rows_ptr,
         float      * dst_ptr,
         float2     * dst_meta_ptr,
         const float scale,
@@ -51,7 +50,6 @@ static __global__ void flash_attn_ext_vec(
     const int  * GGML_CUDA_RESTRICT KV_max   = KV_max_ptr;
     float      * GGML_CUDA_RESTRICT dst      = dst_ptr;
     float2     * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;
-    GGML_UNUSED(KV_rows_ptr);
 
     // Skip unused kernel variants for faster compilation:
     if (use_logit_softcap && !(D == 128 || D == 256)) {
@@ -516,7 +514,7 @@ static __global__ void flash_attn_ext_vec(
         dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]);
     }
 #else
-    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
+    GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,
         max_bias, m0, m1, n_head_log2, logit_softcap,
         ne00, ne01, ne02, ne03,
               nb01, nb02, nb03,

ggml/src/ggml-cuda/fattn.cu

@@ -5,8 +5,7 @@
 #include "fattn-vec.cuh"
 #include "fattn.cuh"
 
-#if !defined(GGML_USE_MUSA)
-
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
 // one list per group of ncols1 queries: a column is selected if any query of the group can see it
 template <int ncols1, bool oob>
 __launch_bounds__(256, 1)
@@ -15,20 +14,10 @@ static __global__ void flash_attn_mask_to_sparse_indices(
         const int n_kv_max, const int64_t s31, const int64_t s33) {
     ggml_cuda_pdl_sync();
 
-    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
     constexpr int values_per_lane = 8;
-    using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
-    constexpr uint64_t ballot_mask = ~(lane_mask_t)0;
-    const auto popc = [](const lane_mask_t mask) {
-        if constexpr (warp_size == 64) {
-            return __popcll(mask);
-        } else {
-            return __popc(mask);
-        }
-    };
     const int tid      = threadIdx.x;
-    const int warp     = tid / warp_size;
-    const int lane     = tid % warp_size;
+    const int warp     = tid / WARP_SIZE;
+    const int lane     = tid % WARP_SIZE;
     const int sequence = blockIdx.y;
     const int group    = blockIdx.x;
 
@@ -38,7 +27,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
     const half * mask = mask_ptr + sequence*s33 + q0*s31;
     int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + group)*n_kv_max;
 
-    __shared__ int warp_offsets[256/warp_size];
+    __shared__ int warp_offsets[256/WARP_SIZE];
     __shared__ int row_count;
     __shared__ int chunk_count;
 
@@ -48,11 +37,11 @@ static __global__ void flash_attn_mask_to_sparse_indices(
     __syncthreads();
 
     for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
-        lane_mask_t selected_warp[values_per_lane];
+        uint32_t selected_warp[values_per_lane];
         int warp_count = 0;
 #pragma unroll
         for (int item = 0; item < values_per_lane; ++item) {
-            const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
+            const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;
             bool selected = false;
             if (i < ne30) {
 #pragma unroll
@@ -60,8 +49,8 @@ static __global__ void flash_attn_mask_to_sparse_indices(
                     selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
                 }
             }
-            selected_warp[item] = (lane_mask_t)__ballot_sync(ballot_mask, (int) selected);
-            warp_count += popc(selected_warp[item]);
+            selected_warp[item] = __ballot_sync(0xFFFFFFFF, selected);
+            warp_count += __popc(selected_warp[item]);
         }
 
         if (lane == 0) {
@@ -72,7 +61,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
         if (tid == 0) {
             int offset = 0;
 #pragma unroll
-            for (int iw = 0; iw < 256/warp_size; ++iw) {
+            for (int iw = 0; iw < 256/WARP_SIZE; ++iw) {
                 const int count = warp_offsets[iw];
                 warp_offsets[iw] = offset;
                 offset += count;
@@ -81,16 +70,16 @@ static __global__ void flash_attn_mask_to_sparse_indices(
         }
         __syncthreads();
 
-        const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1;
+        const uint32_t lane_mask = lane == 0 ? 0 : (1u << lane) - 1;
         int warp_item_offset = 0;
 #pragma unroll
         for (int item = 0; item < values_per_lane; ++item) {
-            const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
-            const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask);
-            if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) {
+            const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;
+            const int dst = row_count + warp_offsets[warp] + warp_item_offset + __popc(selected_warp[item] & lane_mask);
+            if ((selected_warp[item] & (uint32_t(1) << lane)) && dst < n_kv_max) {
                 indices[dst] = i;
             }
-            warp_item_offset += popc(selected_warp[item]);
+            warp_item_offset += __popc(selected_warp[item]);
         }
         __syncthreads();
 
@@ -112,13 +101,13 @@ static __global__ void flash_attn_mask_to_sparse_indices(
     // the dependent grid reads indices, signal once the row is complete
     ggml_cuda_pdl_lc();
 }
-#endif // !defined(GGML_USE_MUSA)
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
 
 void ggml_cuda_flash_attn_ext_compact_mask(
         const ggml_tensor * mask, int32_t * indices, int32_t * counts, int32_t n_queries, int32_t ncols1, int32_t n_kv_max, cudaStream_t stream) {
-#ifdef GGML_USE_MUSA
+#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
     GGML_UNUSED_VARS(mask, indices, counts, n_queries, ncols1, n_kv_max, stream);
-    GGML_ABORT("sparse flash attention is not supported on MUSA");
+    GGML_ABORT("sparse flash attention is only supported on NVIDIA CUDA");
 #else
     const int64_t s31 = mask->nb[1] / sizeof(half);
     const int64_t s33 = mask->nb[3] / sizeof(half);
@@ -133,17 +122,16 @@ void ggml_cuda_flash_attn_ext_compact_mask(
     ggml_cuda_kernel_launch(kernel, launch_params,
         (const half *) mask->data, indices, counts, int(mask->ne[0]), n_queries, n_kv_max, s31, s33);
     CUDA_CHECK(cudaGetLastError());
-#endif // !defined(GGML_USE_MUSA)
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
 }
 
 bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2) {
-#ifdef GGML_USE_MUSA
+#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
     GGML_UNUSED_VARS(cc, dst, ncols1, ncols2);
     return false;
 #else
     const ggml_tensor * Q    = dst->src[0];
     const ggml_tensor * K    = dst->src[1];
-    const ggml_tensor * V    = dst->src[2];
     const ggml_tensor * mask = dst->src[3];
 
     float max_bias = 0.0f;
@@ -151,21 +139,16 @@ bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_
     memcpy(&max_bias,      (const float *) dst->op_params + 1, sizeof(float));
     memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float));
 
-    // On CDNA only the DKQ=576 kernel variant with ncols2=16 is compiled with sparse support.
-    if (amd_mfma_available(cc) && !ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(int(Q->ne[0]), int(V->ne[0]), 1, 16)) {
-        return false;
-    }
-
     const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4);
 
     // the dense kernel handles up to 64/ncols2 queries per K/V pass, the single-query gather has to beat that
     const int64_t n_gather = (ncols1 == 1 ? std::min<int64_t>(Q->ne[1], 64/ncols2) : ncols1) * (int64_t) n_kv_max;
 
-    return ((GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc)) || amd_mfma_available(cc)) && dst->src[5] == nullptr &&
+    return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) &&
         mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f &&
         mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 &&
-        K->ne[1] >= std::max<int64_t>(4096, GGML_CUDA_CC_IS_NVIDIA(cc) ? 2*n_gather : 2LL*n_kv_max);
-#endif // !defined(GGML_USE_MUSA)
+        K->ne[1] >= std::max<int64_t>(4096, 2*n_gather);
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
 }
 
 template <int DKQ, int DV, int ncols2>
@@ -173,7 +156,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con
     const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
     const ggml_tensor * Q = dst->src[0];
 
-#if !defined(GGML_USE_MUSA)
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
     if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 1, ncols2)) {
         // a sparse variant at the full tile width gathers the union of its queries once, prefer it for large batches
         constexpr bool has_wide_sparse = ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 64/ncols2, ncols2);
@@ -182,7 +165,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con
             return;
         }
     }
-#endif // !defined(GGML_USE_MUSA)
+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
 
     if constexpr (ncols2 <= 8) {
         if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) {
@@ -216,15 +199,12 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con
     const ggml_tensor * V    = dst->src[2];
     const ggml_tensor * mask = dst->src[3];
 
-    // with kv_rows the kernel iterates over the mask columns
-    const int64_t n_kv = dst->src[5] ? dst->src[5]->ne[0] : K->ne[1];
-
     float max_bias = 0.0f;
     memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));
 
     // Edge cases like no mask, ALiBi, unpadded K/V, or misaligned addresses for large data transfers
     //     are put into the template specialization without GQA optimizations.
-    bool use_gqa_opt = mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
+    bool use_gqa_opt = mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;
     for (const ggml_tensor * t : {Q, K, V, mask}) {
         if (t == nullptr || ggml_is_quantized(t->type)) {
             continue;
@@ -385,9 +365,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
 
             GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);
             const int gqa_ratio = Q->ne[2] / K->ne[2];
-            if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
-                ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
-            } else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
+            if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
                 if (cc >= GGML_CUDA_CC_DGX_SPARK) {
                     if (Q->ne[1] <= 8) {
                         ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
@@ -570,10 +548,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
     const ggml_tensor * K     = dst->src[1];
     const ggml_tensor * V     = dst->src[2];
     const ggml_tensor * mask  = dst->src[3];
-    const ggml_tensor * kv_rows = dst->src[5];
-
-    // with kv_rows the kernel iterates over the mask columns
-    const int64_t n_kv = kv_rows ? kv_rows->ne[0] : K->ne[1];
 
     const int gqa_ratio = Q->ne[2] / K->ne[2];
     GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);
@@ -583,7 +557,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
 
     // The effective batch size for the kernel can be increased by gqa_ratio.
     // The kernel versions without this optimization are also used for ALiBi, if there is no mask, or if the KV cache is not padded,
-    bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
+    bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0;
     for (const ggml_tensor * t : {Q, K, V, mask}) {
         if (t == nullptr || ggml_is_quantized(t->type)) {
             continue;
@@ -655,12 +629,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
         return BEST_FATTN_KERNEL_NONE;
     }
 
-    // only the MMA kernel reads K/V through kv_rows
-    if (kv_rows) {
-        return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) && mask && Q->ne[0] != 40 && Q->ne[0] != 72 ?
-            BEST_FATTN_KERNEL_MMA_F16 : BEST_FATTN_KERNEL_NONE;
-    }
-
     // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes:
     // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path.
     const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0;
@@ -711,11 +679,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
         return BEST_FATTN_KERNEL_MMA_F16;
     }
 
-    // On CDNA, the sparse MMA kernel is used for DKQ=576 regardless of the head size heuristics.
-    if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
-        return BEST_FATTN_KERNEL_MMA_F16;
-    }
-
     // AMD MFMA needs a certain minimum batch size to outscale the tile kernel for large head sizes.
     if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
         if ((Q->ne[0] <= 64 && Q->ne[1] * gqa_ratio_eff > 8)) {

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -9,7 +9,6 @@
 #include "ggml-cuda/arange.cuh"
 #include "ggml-cuda/argmax.cuh"
 #include "ggml-cuda/argsort.cuh"
-#include "ggml-cuda/batch-get.cuh"
 #include "ggml-cuda/binbcast.cuh"
 #include "ggml-cuda/clamp.cuh"
 #include "ggml-cuda/col2im-1d.cuh"
@@ -862,51 +861,6 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
     CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
 }
 
-static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
-    ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
-
-    ggml_cuda_set_device(ctx->device);
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        if (sizes[i] == 0) {
-            continue;
-        }
-        CUDA_CHECK(cudaMemcpyAsync(
-            (char *) tensors[i]->data + offsets[i],
-            data[i],
-            sizes[i],
-            cudaMemcpyHostToDevice,
-            cudaStreamPerThread));
-    }
-
-    CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
-}
-
-static void ggml_backend_cuda_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
-    ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
-
-    ggml_cuda_set_device(ctx->device);
-
-    if (!ggml_cuda_pinned_memory_disabled() &&
-            ggml_cuda_get_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) {
-        return;
-    }
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        if (sizes[i] == 0) {
-            continue;
-        }
-        CUDA_CHECK(cudaMemcpyAsync(
-            data[i],
-            (const char *) tensors[i]->data + offsets[i],
-            sizes[i],
-            cudaMemcpyDeviceToHost,
-            cudaStreamPerThread));
-    }
-
-    CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
-}
-
 static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
     if (ggml_backend_buffer_is_cuda(src->buffer)) {
         ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
@@ -932,54 +886,6 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
     GGML_UNUSED(buffer);
 }
 
-static bool ggml_backend_cuda_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) {
-    ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
-        if (!ggml_backend_buffer_is_cuda(src_buf)) {
-            return false;
-        }
-    }
-
-#ifdef GGML_CUDA_NO_PEER_COPY
-    for (size_t i = 0; i < n_tensors; i++) {
-        ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
-        ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
-        ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src_buf->context;
-        ggml_backend_cuda_buffer_context * dst_ctx_i = (ggml_backend_cuda_buffer_context *)dst_buf->context;
-        const int src_physical = ggml_cuda_get_physical_device(src_ctx->device);
-        const int dst_physical = ggml_cuda_get_physical_device(dst_ctx_i->device);
-        if (src_physical != dst_physical) {
-            return false;
-        }
-    }
-#endif
-
-    ggml_cuda_set_device(dst_ctx->device);
-
-    for (size_t i = 0; i < n_tensors; i++) {
-        ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
-        ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
-        ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src_buf->context;
-        ggml_backend_cuda_buffer_context * dst_ctx_i = (ggml_backend_cuda_buffer_context *)dst_buf->context;
-        const int src_physical = ggml_cuda_get_physical_device(src_ctx->device);
-        const int dst_physical = ggml_cuda_get_physical_device(dst_ctx_i->device);
-        if (src_physical == dst_physical) {
-            CUDA_CHECK(cudaMemcpyAsync(dsts[i]->data, srcs[i]->data, ggml_nbytes(srcs[i]), cudaMemcpyDeviceToDevice, cudaStreamPerThread));
-        } else {
-#ifdef GGML_CUDA_NO_PEER_COPY
-            GGML_ABORT("peer copy not supported");
-#else
-            CUDA_CHECK(cudaMemcpyPeerAsync(dsts[i]->data, dst_physical, srcs[i]->data, src_physical, ggml_nbytes(srcs[i]), cudaStreamPerThread));
-#endif
-        }
-    }
-
-    CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
-    return true;
-}
-
 static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
     ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
 
@@ -1000,9 +906,6 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
     /* .cpy_tensor      = */ ggml_backend_cuda_buffer_cpy_tensor,
     /* .clear           = */ ggml_backend_cuda_buffer_clear,
     /* .reset           = */ NULL,
-    /* .set_tensor_n    = */ ggml_backend_cuda_buffer_set_tensor_n,
-    /* .get_tensor_n    = */ ggml_backend_cuda_buffer_get_tensor_n,
-    /* .cpy_tensor_n    = */ ggml_backend_cuda_buffer_cpy_tensor_n,
 };
 
 // cuda buffer type
@@ -1123,18 +1026,6 @@ struct ggml_backend_cuda_comm_context {
 
     ggml_cuda_ar_pipeline *     ar_pipeline = nullptr;
 
-    // Number of ranks in the communicator when each rank is a separate process. Zero unless
-    // comm_init_rank created this context.
-    int                         world_size = 0;
-
-    // Number of ranks taking part in the reduction. For a communicator from comm_init_rank this is
-    // world_size. Otherwise it is the number of backends this process holds. The NCCL reduction
-    // uses it to choose the tensor size below which it reduces in F32 rather than converting to
-    // BF16 first.
-    size_t n_ranks() const {
-        return world_size > 0 ? (size_t) world_size : backends.size();
-    }
-
 #ifdef GGML_USE_NCCL
     std::vector<ncclComm_t>     comms;
 #endif // GGML_USE_NCCL
@@ -1162,7 +1053,6 @@ static bool ggml_backend_cuda_comm_allreduce_nccl(
     }
 
     const size_t n_backends = comm_ctx->backends.size();
-    const size_t n_ranks    = comm_ctx->n_ranks();
 
     for (size_t i = 0; i < n_backends; ++i) {
         GGML_ASSERT(tensors[i] != nullptr);
@@ -1172,7 +1062,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl(
 
     // For small tensors, simply reduce them as FP32.
     // The following heuristic for how "small" a tensor should be is based on RTX 4090s connected via 16x PCIe 4.0.
-    if ((n_ranks <= 2 && ne < 32768) || (n_ranks == 3 && ne < 131072) || (n_ranks >= 4 && ne < 262144)) {
+    if ((n_backends <= 2 && ne < 32768) || (n_backends == 3 && ne < 131072) || (n_backends >= 4 && ne < 262144)) {
         for (size_t i = 0; i < n_backends; ++i) {
             if ((tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
                 ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) comm_ctx->backends[i]->context;
@@ -1403,90 +1293,6 @@ static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_ba
     return ret;
 }
 
-#ifdef GGML_USE_NCCL
-// Builds the communicator with ncclCommInitRank, for the case where each rank is its own
-// process. Blocks until all world_size ranks have joined.
-static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank, int world_size,
-                                               ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data) {
-    if (world_size < 1 || rank < 0 || rank >= world_size) {
-        GGML_LOG_ERROR("%s: invalid rank %d for world size %d\n", __func__, rank, world_size);
-        return nullptr;
-    }
-
-    // Every rank sends whether its local checks passed, and rank 0 also sends the NCCL unique id.
-    // So all ranks either join together or fail together, and no rank waits in ncclCommInitRank alone.
-    struct bootstrap_msg {
-        uint8_t      ok;
-        ncclUniqueId id;
-    };
-    bootstrap_msg local;
-    memset(&local, 0, sizeof(local));
-    local.ok = 1;
-
-    if (!ggml_backend_is_cuda(backend)) {
-        GGML_LOG_ERROR("%s: backend is not a CUDA backend\n", __func__);
-        local.ok = 0;
-    } else {
-        // NCCL requires one distinct physical GPU per rank, see ggml_backend_cuda_comm_init_nccl.
-        const ggml_cuda_device_info & info = ggml_cuda_info();
-        if (info.device_count > info.physical_device_count) {
-            GGML_LOG_ERROR("%s: CUDA virtual devices are in use, which NCCL does not support\n", __func__);
-            local.ok = 0;
-        }
-    }
-    if (local.ok && rank == 0) {
-        const ncclResult_t rc = ncclGetUniqueId(&local.id);
-        if (rc != ncclSuccess) {
-            GGML_LOG_ERROR("%s: ncclGetUniqueId failed (%s)\n", __func__, ncclGetErrorString(rc));
-            local.ok = 0;
-        }
-    }
-
-    std::vector<bootstrap_msg> all(world_size);
-    if (!bootstrap_callback(bootstrap_callback_data, &local, all.data(), sizeof(local))) {
-        GGML_LOG_ERROR("%s: bootstrap exchange failed on rank %d\n", __func__, rank);
-        return nullptr;
-    }
-    for (int i = 0; i < world_size; i++) {
-        if (!all[i].ok) {
-            GGML_LOG_ERROR("%s: rank %d failed its setup checks\n", __func__, i);
-            return nullptr;
-        }
-    }
-
-    ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
-
-    auto * ret = new ggml_backend_cuda_comm_context;
-    ret->backends.push_back(backend);
-    ret->dev_ids.push_back(cuda_ctx->device);
-    ret->world_size = world_size;
-    ret->comms.resize(1);
-    ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_nccl;
-
-    // On DGX Spark, NCCL picks 64 channels by default, which is slower and uses more memory than 8.
-    // NCCL_MAX_CTAS still overrides this.
-    ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
-    const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
-    if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
-        config.maxCTAs = 8;
-    }
-
-    ggml_cuda_set_device(cuda_ctx->device);
-    const ncclResult_t rc = ncclCommInitRankConfig(&ret->comms[0], world_size, all[0].id, rank, &config);
-    if (rc != ncclSuccess) {
-        GGML_LOG_ERROR("%s: ncclCommInitRankConfig failed for rank %d/%d (%s)\n",
-                       __func__, rank, world_size, ncclGetErrorString(rc));
-        ret->comms.clear();
-        delete ret;
-        return nullptr;
-    }
-
-    GGML_LOG_INFO("%s: joined NCCL communicator as rank %d/%d on device %d\n",
-                  __func__, rank, world_size, cuda_ctx->device);
-    return ret;
-}
-#endif // GGML_USE_NCCL
-
 // Top-level dispatch -- calls the function pointer chosen by comm_init.
 // Returns false to let the meta-backend's butterfly run.
 static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) {
@@ -1759,7 +1565,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
     if (compute_type == GGML_TYPE_F16) {
         prefer_f32_output = cc == GGML_CUDA_CC_VOLTA || GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_CDNA(cc);
     } else if (compute_type == GGML_TYPE_BF16) {
-        prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_RDNA4_M(cc) && !GGML_CUDA_CC_IS_CDNA(cc);
+        prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc);
     }
 
     if (prefer_f32_output) {
@@ -2156,7 +1962,7 @@ static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggm
     const ggml_tensor * x_up = up->src[0];
     const ggml_tensor * y    = up->src[1];
 
-    if ((up->op != GGML_OP_MUL_MAT && up->op != GGML_OP_MUL_MAT_ID) || !ggml_cuda_mmq_fusion_supported(x_up->type) || y->type != GGML_TYPE_F32) {
+    if (up->op != GGML_OP_MUL_MAT || !ggml_cuda_mmq_fusion_supported(x_up->type) || y->type != GGML_TYPE_F32) {
         return false;
     }
 
@@ -2198,18 +2004,12 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t
     const ggml_tensor * x_up = up->src[0];
     const ggml_tensor * y    = up->src[1];
     const int           cc   = ggml_cuda_info().devices[device].cc;
-
-    // For MUL_MAT_ID: same kernel choice as ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
-    const bool    is_id     = up->op == GGML_OP_MUL_MAT_ID;
-    const int64_t ncols     = is_id ? y->ne[2]    : y->ne[1];
-    const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3];
-    const bool    use_mmvq  = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols);
-    if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) {
+    if (ggml_cuda_should_use_mmvq(x_up->type, cc, y->ne[1]) || !ggml_cuda_should_use_mmq(x_up->type, cc, y->ne[1], 0)) {
         return false;
     }
 
     // The fused kernel needs one block per tile (no stream-k split), so only fuse if the tiling is efficient.
-    return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], ncols, nchannels, device);
+    return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], y->ne[1], y->ne[2]*y->ne[3], device);
 }
 
 static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
@@ -6487,11 +6287,6 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con
     if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) {
         return (void *)ggml_backend_cuda_comm_allreduce_tensor;
     }
-#ifdef GGML_USE_NCCL
-    if (strcmp(name, "ggml_backend_comm_init_rank") == 0) {
-        return (void *)ggml_backend_cuda_comm_init_rank;
-    }
-#endif // GGML_USE_NCCL
     if (strcmp(name, "ggml_backend_register_host_buffer") == 0) {
         return (void *)ggml_backend_cuda_register_host_buffer;
     }

ggml/src/ggml-cuda/mma.cuh

@@ -76,11 +76,11 @@ namespace ggml_cuda_mma {
         // For the A/C matrices this means I major == row major, J major == column major.
         // For the B matrix this means I major == column major, J major == row major.
         // MIRRORED == Each data value is held exactly once per thread subgroup.
-        DATA_LAYOUT_I_MAJOR           =  0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for WMMA128 and CDNA.
-        DATA_LAYOUT_J_MAJOR           = 10, // Matrix C for CDNA and WMMA128, int and float matrix C for WMMA256.
+        DATA_LAYOUT_I_MAJOR           =  0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for RDNA4 and CDNA.
+        DATA_LAYOUT_J_MAJOR           = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3.
         DATA_LAYOUT_I_MAJOR_MIRRORED  = 20, // Volta, matrix A&B for RDNA3.
         DATA_LAYOUT_J_MAJOR_MIRRORED  = 30,
-        DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (WMMA128/CDNA), convert to float to unscramble.
+        DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble.
     };
     // Implemented mma combinations are:
     //   - (I_MAJOR, I_MAJOR)          -> I_MAJOR
@@ -88,11 +88,11 @@ namespace ggml_cuda_mma {
     //   - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR
 
     static constexpr __device__ data_layout get_input_data_layout() {
-#if defined(AMD_WMMA_256B_AVAILABLE) || defined(VOLTA_MMA_AVAILABLE)
+#if defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)
         return DATA_LAYOUT_I_MAJOR_MIRRORED;
 #else
         return DATA_LAYOUT_I_MAJOR;
-#endif // defined(AMD_WMMA_256B_AVAILABLE) || defined(VOLTA_MMA_AVAILABLE)
+#endif // defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)
     }
 
     template <int I_, int J_, typename T, data_layout ds_=DATA_LAYOUT_I_MAJOR>
@@ -201,7 +201,7 @@ namespace ggml_cuda_mma {
 
         static __device__ __forceinline__ int get_j(const int l) {
             if constexpr (I == 16 && J == 16) {
-#if defined(AMD_WMMA_256B_AVAILABLE)
+#if defined(RDNA3)
                 if constexpr (std::is_same_v<T, float> || std::is_same_v<T, int>) {
                     // matrix C
                     return 2 * l + (threadIdx.x / 16);
@@ -210,11 +210,11 @@ namespace ggml_cuda_mma {
                     return l;
                 }
 #else
-                // matrix C is the transposed matrix A&B on WMMA128
+                // matrix C is the transposed matrix A&B on RDNA4
                 return ne * (threadIdx.x / 16) + l;
-#endif // defined(AMD_WMMA_256B_AVAILABLE)
+#endif // defined(RDNA3)
             } else if constexpr (I == 16 && J == 8) {
-                // mmq input for WMMA128
+                // mmq input for RDNA4
                 return ne * (threadIdx.x / 16) + l;
             } else if constexpr (I == 16 && J == 4) {
                 return ne * (threadIdx.x / 16) + l;
@@ -573,7 +573,7 @@ namespace ggml_cuda_mma {
         static constexpr int         I  = I_;
         static constexpr int         J  = J_;
         static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;
-#if defined(AMD_WMMA_256B_AVAILABLE)
+#if defined(RDNA3)
         static constexpr int         ne = tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::ne;
 
         half2 x[ne] = {{0.0f, 0.0f}};
@@ -616,7 +616,7 @@ namespace ggml_cuda_mma {
                 return -1;
             }
         }
-#endif // defined(AMD_WMMA_256B_AVAILABLE)
+#endif // defined(RDNA3)
     };
 
     template <int I_, int J_>
@@ -694,7 +694,7 @@ namespace ggml_cuda_mma {
     };
 
     static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) {
-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
+#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
         tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret;
 #pragma unroll
         for (int l0 = 0; l0 < t.ne/2; ++l0) {
@@ -705,7 +705,7 @@ namespace ggml_cuda_mma {
 #else
         NO_DEVICE_CODE;
         GGML_UNUSED(t);
-#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
+#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
     }
 
 #if defined(TURING_MMA_AVAILABLE)
@@ -726,7 +726,7 @@ namespace ggml_cuda_mma {
 
         return ret;
     }
-#elif defined(AMD_WMMA_256B_AVAILABLE)
+#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
     static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2(
             const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) {
         tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret;
@@ -1171,7 +1171,7 @@ namespace ggml_cuda_mma {
             : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));
 #endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
 #elif defined(AMD_WMMA_AVAILABLE)
-#if defined(AMD_WMMA_128B_AVAILABLE)
+#if defined(RDNA4)
         using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;
         halfx8_t& acc_frag = reinterpret_cast<halfx8_t&>(D.x[0]);
         const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);
@@ -1180,7 +1180,7 @@ namespace ggml_cuda_mma {
 #else
         GGML_UNUSED_VARS(D, A, B);
         NO_DEVICE_CODE;
-#endif // defined(AMD_WMMA_128B_AVAILABLE)
+#endif // defined(RDNA4)
 #elif defined(AMD_MFMA_AVAILABLE)
         // MFMA: FP16 input, FP32 accumulate, convert back to half2.
         using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;
@@ -1218,7 +1218,7 @@ namespace ggml_cuda_mma {
     static __device__ __forceinline__ void mma(
             tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A,
             const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) {
-#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
+#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
         tile<16, 8, half2>       * D16 = (tile<16, 8, half2>       *) &D;
         const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A;
         mma(D16[0], A16[0], B);
@@ -1226,7 +1226,7 @@ namespace ggml_cuda_mma {
 #else
         GGML_UNUSED_VARS(D, A, B);
         NO_DEVICE_CODE;
-#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)
     }
 
     template <data_layout dl_ab, data_layout dl_d>
@@ -1373,14 +1373,14 @@ namespace ggml_cuda_mma {
             : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));
 #endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
 #elif defined(AMD_WMMA_AVAILABLE)
-#if defined(AMD_WMMA_128B_AVAILABLE)
+#if defined(RDNA4)
         using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;
         using floatx8_t = __attribute__((ext_vector_type(8))) float;
         floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
         const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);
         const halfx8_t& b_frag = reinterpret_cast<const halfx8_t&>(B.x[0]);
         acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);
-#elif defined(AMD_WMMA_256B_AVAILABLE)
+#elif defined(RDNA3)
         using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;
         using floatx8_t = __attribute__((ext_vector_type(8))) float;
         floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
@@ -1390,7 +1390,7 @@ namespace ggml_cuda_mma {
 #else
         GGML_UNUSED_VARS(D, A, B);
         NO_DEVICE_CODE;
-#endif // AMD_WMMA_128B_AVAILABLE
+#endif // RDNA4
 #elif defined(AMD_MFMA_AVAILABLE)
         using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;
         using floatx4_t = __attribute__((ext_vector_type(4))) float;
@@ -1408,14 +1408,14 @@ namespace ggml_cuda_mma {
     static __device__ __forceinline__ void mma(
             tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) {
 #if defined(AMD_WMMA_AVAILABLE)
-#if defined(AMD_WMMA_128B_AVAILABLE)
+#if defined(RDNA4)
         using bf16x8_t = __attribute__((ext_vector_type(8))) short;
         using floatx8_t = __attribute__((ext_vector_type(8))) float;
         floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
         const bf16x8_t& a_frag = reinterpret_cast<const bf16x8_t&>(A.x[0]);
         const bf16x8_t& b_frag = reinterpret_cast<const bf16x8_t&>(B.x[0]);
         acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag);
-#elif defined(AMD_WMMA_256B_AVAILABLE)
+#elif defined(RDNA3)
         using bf16x16_t = __attribute__((ext_vector_type(16))) short;
         using floatx8_t = __attribute__((ext_vector_type(8))) float;
         floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
@@ -1425,7 +1425,7 @@ namespace ggml_cuda_mma {
 #else
         GGML_UNUSED_VARS(D, A, B);
         NO_DEVICE_CODE;
-#endif // defined(AMD_WMMA_128B_AVAILABLE)
+#endif // defined(RDNA4)
 #elif defined(AMD_MFMA_AVAILABLE)
         using floatx4_t = __attribute__((ext_vector_type(4))) float;
         floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);
@@ -1467,19 +1467,19 @@ namespace ggml_cuda_mma {
 #elif defined(AMD_WMMA_AVAILABLE)
         using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;
         int32x8_t * acc = (int32x8_t *) D.x;
-#if defined(AMD_WMMA_128B_AVAILABLE)
+#if defined(RDNA4)
         using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;
         int32x2_t * a_vec = (int32x2_t *) A.x;
         int32x2_t * b_vec = (int32x2_t *) B.x;
         acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], true);
         acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[1], true, b_vec[1], acc[0], true);
-#elif defined(AMD_WMMA_256B_AVAILABLE)
+#elif defined(RDNA3)
         using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
         int32x4_t * a_vec = (int32x4_t *) A.x;
         int32x4_t * b_vec = (int32x4_t *) B.x;
         acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], true);
         acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[1], true, b_vec[1], acc[0], true);
-#endif // AMD_WMMA_128B_AVAILABLE
+#endif // RDNA4
 #else
         GGML_UNUSED_VARS(D, A, B);
         NO_DEVICE_CODE;
@@ -1556,7 +1556,7 @@ namespace ggml_cuda_mma {
     static __device__ __forceinline__ void mma(
             tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A,
             const tile<16,  8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {
-#if defined(AMD_WMMA_256B_AVAILABLE)
+#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
         using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;
         halfx16_t       * xD = (halfx16_t       *) D.x;
         const halfx16_t * xA = (const halfx16_t *) A.x;
@@ -1585,17 +1585,17 @@ namespace ggml_cuda_mma {
 #elif defined(AMD_WMMA_AVAILABLE)
         using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;
         int32x8_t * acc = (int32x8_t *) D.x;
-#if defined(AMD_WMMA_128B_AVAILABLE)
+#if defined(RDNA4)
         using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;
         int32x2_t * a_vec = (int32x2_t *) A.x;
         int32x2_t * b_vec = (int32x2_t *) B.x;
         acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], false);
-#elif defined(AMD_WMMA_256B_AVAILABLE)
+#elif defined(RDNA3)
         using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
         int32x4_t * a_vec = (int32x4_t *) A.x;
         int32x4_t * b_vec = (int32x4_t *) B.x;
         acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], false);
-#endif // AMD_WMMA_128B_AVAILABLE
+#endif // RDNA4
 #else
         GGML_UNUSED(D);
         GGML_UNUSED(A);

ggml/src/ggml-cuda/mmq.cu

@@ -170,13 +170,10 @@ void ggml_cuda_mul_mat_q(
 
     // Fused gate/up/GLU: src0 is the up weight, dst the GLU output.
     const ggml_tensor * gate = fusion ? fusion->gate : nullptr;
-    ggml_cuda_mm_fusion_args_device fusion_device = {};
     if (fusion) {
-        GGML_ASSERT(gate && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale);
+        // TODO MoE support
+        GGML_ASSERT(gate && !ids && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale);
         GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0));
-        fusion_device.gate      = gate->data;
-        fusion_device.glu_op    = fusion->glu_op;
-        fusion_device.glu_limit = fusion->glu_limit;
     }
 
     GGML_TENSOR_BINARY_OP_LOCALS;
@@ -242,7 +239,7 @@ void ggml_cuda_mul_mat_q(
 
             // Each expert only sees ne12*n_expert_used/ne02 tokens on average.
             // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.
-            if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {
+            if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {
                 ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02;
             }
         }
@@ -289,6 +286,13 @@ void ggml_cuda_mul_mat_q(
                                 ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));
         const int64_t s13 = ne12*s12;
 
+        ggml_cuda_mm_fusion_args_device fusion_device = {};
+        if (gate) {
+            fusion_device.gate      = gate->data;
+            fusion_device.glu_op    = fusion->glu_op;
+            fusion_device.glu_limit = fusion->glu_limit;
+        }
+
         const mmq_args args = {
             src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d,
             src0->type == GGML_TYPE_NVFP4 && use_native_fp4 ? src1_scale.ptr : nullptr,
@@ -374,7 +378,7 @@ void ggml_cuda_mul_mat_q(
         ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,
         ne02, ne02, s02, s12, s2,
         ne03, ne13, s03, s13, s3,
-        ne12, J_best, fusion_device};
+        ne12, J_best};
 
     ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
 }
@@ -470,7 +474,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
     }
 
     if (amd_wmma_available(cc)) {
-        if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
+        if (GGML_CUDA_CC_IS_RDNA3(cc)) {
             // High expert counts are almost always better on MMQ due to
             //     the synchronization overhead in the cuBLAS/hipBLAS path:
             // https://github.com/ggml-org/llama.cpp/pull/18202
@@ -487,7 +491,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
                     return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);
                 case GGML_TYPE_IQ2_XS:
                 case GGML_TYPE_IQ2_S:
-                    return GGML_CUDA_CC_IS_RDNA3_5(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || ne11 <= 128;
+                    return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128;
                 default:
                     return true;
             }

ggml/src/ggml-cuda/mmq.cuh

@@ -240,9 +240,6 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty
         if (GGML_CUDA_CC_IS_RDNA4(cc)) {
             return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
         }
-        if (GGML_CUDA_CC_IS_RDNA4_M(cc)) {
-            return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
-        }
         if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
             return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
         }
@@ -275,8 +272,6 @@ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_t
     return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
 #elif defined(RDNA4)
     return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
-#elif defined(RDNA4_M)
-    return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
 #elif defined(RDNA3_5)
     return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
 #elif defined(RDNA3)

ggml/src/ggml-cuda/mmvf.cu

@@ -844,7 +844,7 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const
                     return ne11 <= MMVF_MAX_BATCH_SIZE;
                 }
                 if (fp16_mma_hardware_available(cc)) {
-                    if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
+                    if (GGML_CUDA_CC_IS_RDNA3(cc)) {
                         return ne11 <= 3;
                     }
                     if (GGML_CUDA_CC_IS_RDNA4(cc)) {

ggml/src/ggml-cuda/mmvq.cu

@@ -108,7 +108,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
     return MMVQ_PARAMETERS_RDNA4;
 #elif defined(RDNA3_0)
     return MMVQ_PARAMETERS_RDNA3_0;
-#elif defined(RDNA2) || defined(RDNA3_5) || defined(RDNA4_M)
+#elif defined(RDNA2) || defined(RDNA3_5)
     return MMVQ_PARAMETERS_RDNA2;
 #elif defined(GCN) || defined(CDNA)
     return MMVQ_PARAMETERS_GCN;
@@ -128,7 +128,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
     if (GGML_CUDA_CC_IS_RDNA3_0(cc)) {
         return MMVQ_PARAMETERS_RDNA3_0;
     }
-    if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
+    if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) {
         return MMVQ_PARAMETERS_RDNA2;
     }
     if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {
@@ -299,7 +299,7 @@ int get_mmvq_mmid_max_batch(ggml_type type, int cc) {
         if (GGML_CUDA_CC_IS_RDNA4(cc)) {
             return get_mmvq_mmid_max_batch_rdna4(type);
         }
-        if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
+        if (GGML_CUDA_CC_IS_RDNA3(cc)) {
             return get_mmvq_mmid_max_batch_rdna3(type);
         }
         if (GGML_CUDA_CC_IS_RDNA1(cc) || GGML_CUDA_CC_IS_RDNA2(cc)) {
@@ -432,7 +432,7 @@ template <ggml_type type>
 static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {
 #if defined(RDNA4)
     return get_mmvq_mmid_max_batch_rdna4(type);
-#elif defined(RDNA3) || defined(RDNA4_M)
+#elif defined(RDNA3)
     return get_mmvq_mmid_max_batch_rdna3(type);
 #elif defined(RDNA2) || defined(RDNA1)
     return get_mmvq_mmid_max_batch_rdna1_rdna2(type);

ggml/src/ggml-cuda/quantize.cu

@@ -83,21 +83,21 @@ static __global__ void quantize_q8_1(
     ggml_cuda_pdl_sync();
     const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f;
     float amax = fabsf(xi);
+    float sum = xi;
 
     amax = warp_reduce_max<QK8_1>(amax);
+    sum  = warp_reduce_sum<QK8_1>(sum);
 
     const float  d = amax / 127.0f;
     const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
 
-    const int sum = warp_reduce_sum<QK8_1>((int) q);
-
     y[ib].qs[iqs] = q;
 
     if (iqs > 0) {
         return;
     }
 
-    y[ib].ds = make_half2(d, d * sum);
+    y[ib].ds = make_half2(d, sum);
 }
 
 __device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) {
@@ -500,6 +500,17 @@ static __global__ void quantize_mmq_q8_1(
         amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
     }
 
+    float sum;
+    if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
+        sum = xi.x + xi.y + xi.z + xi.w;
+
+        // Calculate sums across vals_per_sum/4 threads.
+#pragma unroll
+        for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {
+            sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE);
+        }
+    }
+
     const float d_inv = 127.0f / amax;
     char4 q;
     q.x = roundf(xi.x*d_inv);
@@ -508,19 +519,6 @@ static __global__ void quantize_mmq_q8_1(
     q.w = roundf(xi.w*d_inv);
     const float d = 1.0f / d_inv;
 
-    float sum = 0.0f;
-    if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
-        int sum_q = int(q.x) + int(q.y) + int(q.z) + int(q.w);
-
-        // Calculate sums across vals_per_sum/4 threads.
-#pragma unroll
-        for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {
-            sum_q += __shfl_xor_sync(0xFFFFFFFF, sum_q, offset, WARP_SIZE);
-        }
-
-        sum = d * float(sum_q);
-    }
-
     // write the block once (normal) or to each of the token's compact rows (scatter)
     const int nwrite = scatter ? n_expert_used : 1;
 #pragma unroll

ggml/src/ggml-cuda/topk-moe.cu

@@ -402,10 +402,6 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx,
             ggml_cuda_kernel_launch(topk_moe_cuda<288, has_bias>, launch_params,
                 logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
             break;
-        case 384: // DeepSeek V4.1
-            ggml_cuda_kernel_launch(topk_moe_cuda<384, has_bias>, launch_params,
-                logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
-            break;
         case 512:
             ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params,
                 logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
@@ -476,7 +472,7 @@ bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op,
     // must match an instantiation of launch_topk_moe_cuda: a power of 2 up to 512,
     // or one of the non-power-of-2 expert counts of supported models
     const int n_expert = ids->nb[1] / ids->nb[0];
-    if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 384 && n_expert != 576) {
+    if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 576) {
         return false;
     }
 

ggml/src/ggml-cuda/vendors/hip.h

@@ -216,13 +216,9 @@
 #define RDNA4
 #endif // defined(__GFX12__)
 
-#if defined(__gfx1170__) || defined(__gfx1171__)
-#define RDNA4_M
-#endif // defined(__gfx1170__) || defined(__gfx1171__)
-
-#if defined(__GFX11__) && !defined(RDNA4_M)
+#if defined(__GFX11__)
 #define RDNA3
-#endif // defined(__GFX11__) && !defined(RDNA4_M)
+#endif // defined(__GFX11__)
 
 #if defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)
 #define RDNA3_5
@@ -241,9 +237,9 @@
 #define RDNA1
 #endif // defined(__gfx1010__) || defined(__gfx1012__)
 
-#if defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
+#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
 #define RDNA // For the entire family
-#endif // defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
+#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
 
 #ifndef __has_builtin
     #define __has_builtin(x) 0
@@ -258,10 +254,6 @@ typedef __hip_fp8_e4m3 __nv_fp8_e4m3;
 #define FP8_AVAILABLE
 #endif // HIP_VERSION >= 60300000
 
-#if HIP_VERSION < 70000000
-#define __ballot_sync(mask, var) __ballot(var)
-#endif //HIP_VERSION < 70000000
-
 typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
 typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));
 static __device__ __forceinline__ int __vsubss4(const int a, const int b) {

ggml/src/ggml-et/ggml-et.cpp

@@ -373,9 +373,6 @@ static const struct ggml_backend_buffer_i ggml_backend_et_buffer_i = {
     /* .cpy_tensor      = */ ggml_backend_et_buffer_cpy_tensor,
     /* .clear           = */ ggml_backend_et_buffer_clear,
     /* .reset           = */ NULL,
-    /* .set_tensor_n    = */ NULL,
-    /* .get_tensor_n    = */ NULL,
-    /* .cpy_tensor_n    = */ NULL,
 };
 
 static const char * ggml_backend_et_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
@@ -1273,7 +1270,7 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm
         case GGML_OP_FLASH_ATTN_EXT:
             if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
                 (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && op->src[2] &&
-                (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr && op->src[5] == nullptr &&
+                (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr &&
                 ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0])) {
                 float max_bias      = 0.0f;
                 float logit_softcap = 0.0f;