mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-10-30 08:27:34 -05:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Aman Gupta
|
b431451636 | ||
|
Aman Gupta
|
d88ec8cc70 | ||
|
Aman Gupta
|
d80fe6bea3 |
@@ -13,8 +13,6 @@ on:
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.py',
|
||||
'**/requirements*.txt',
|
||||
]
|
||||
|
||||
pull_request:
|
||||
@@ -26,9 +24,7 @@ on:
|
||||
'**/*.h',
|
||||
'**/*.hpp',
|
||||
'**/*.c',
|
||||
'**/*.cpp',
|
||||
'**/*.py',
|
||||
'**/requirements*.txt',
|
||||
'**/*.cpp'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
@@ -73,7 +69,7 @@ jobs:
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
pip install jinja2==3.1.6
|
||||
|
||||
- name: ccache-buckets-restore
|
||||
uses: ./.github/actions/ccache-buckets
|
||||
@@ -138,7 +134,7 @@ jobs:
|
||||
run: |
|
||||
python3 -m venv .venv
|
||||
source .venv/Scripts/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
pip install jinja2==3.1.6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -160,13 +156,3 @@ jobs:
|
||||
source .venv/Scripts/activate
|
||||
cd build
|
||||
ctest -C Release -L behavior -LE 'model|backend' --verbose --timeout 900
|
||||
|
||||
- name: ccache-clear
|
||||
uses: ./.github/actions/ccache-clear
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
key: behavior-check-windows-2025
|
||||
older: 5m
|
||||
min: 1
|
||||
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
name: CI (apple - daily)
|
||||
|
||||
on:
|
||||
workflow_dispatch: # allows manual triggering
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GGML_NLOOP: 3
|
||||
GGML_N_THREADS: 1
|
||||
LLAMA_ARG_LOG_COLORS: 1
|
||||
LLAMA_ARG_LOG_PREFIX: 1
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
macos-latest-ios-xcode:
|
||||
runs-on: macos-latest
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Xcode
|
||||
uses: ggml-org/setup-xcode@v1
|
||||
with:
|
||||
xcode-version: latest-stable
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
- name: xcodebuild for swift package
|
||||
id: xcodebuild
|
||||
run: |
|
||||
./build-xcframework.sh
|
||||
|
||||
- name: Upload xcframework artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
retention-days: 1
|
||||
|
||||
- name: Build Xcode project
|
||||
run: |
|
||||
xcodebuild -downloadPlatform iOS
|
||||
xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
||||
|
||||
macos-latest-tvos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-visionos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-swift:
|
||||
runs-on: macos-latest
|
||||
needs: macos-latest-ios-xcode
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download xcframework artifact
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
|
||||
- name: Build llama.cpp with CMake
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
@@ -107,3 +107,140 @@ jobs:
|
||||
# ref: https://github.com/ggml-org/llama.cpp/pull/19802#issuecomment-4013704023
|
||||
ctest -L main -E "test-llama-archs|test-llama-context|test-speculative-rejection-sampling" --verbose --timeout 900
|
||||
|
||||
macos-latest-ios-xcode:
|
||||
runs-on: macos-latest
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Xcode
|
||||
uses: ggml-org/setup-xcode@v1
|
||||
with:
|
||||
xcode-version: latest-stable
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=iOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
- name: xcodebuild for swift package
|
||||
id: xcodebuild
|
||||
run: |
|
||||
./build-xcframework.sh
|
||||
|
||||
- name: Upload xcframework artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
retention-days: 1
|
||||
|
||||
- name: Build Xcode project
|
||||
run: |
|
||||
xcodebuild -downloadPlatform iOS
|
||||
xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
||||
|
||||
macos-latest-tvos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=tvOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-visionos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_BUILD_COMMON=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_SYSTEM_NAME=visionOS \
|
||||
-DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
|
||||
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
||||
|
||||
macos-latest-swift:
|
||||
runs-on: macos-latest
|
||||
needs: macos-latest-ios-xcode
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download xcframework artifact
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: llama-xcframework
|
||||
path: build-apple/llama.xcframework/
|
||||
|
||||
- name: Build llama.cpp with CMake
|
||||
id: cmake_build
|
||||
run: |
|
||||
sysctl -a
|
||||
cmake -B build -G Xcode \
|
||||
-DGGML_METAL_EMBED_LIBRARY=ON \
|
||||
-DLLAMA_OPENSSL=OFF \
|
||||
-DLLAMA_BUILD_APP=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=OFF \
|
||||
-DLLAMA_BUILD_TOOLS=OFF \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_SERVER=OFF \
|
||||
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
||||
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
||||
|
||||
@@ -43,9 +43,9 @@ env:
|
||||
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
||||
|
||||
jobs:
|
||||
ubuntu-26-s390x:
|
||||
name: ubuntu-26-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
ubuntu-24-s390x:
|
||||
name: ubuntu-24-s390x (VXE ${{ matrix.vxe }})
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -66,6 +66,12 @@ jobs:
|
||||
libjpeg-dev build-essential libssl-dev \
|
||||
git-lfs
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
run: |
|
||||
sudo apt-get install -y gcc-14 g++-14
|
||||
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Python Dependencies
|
||||
id: python_depends
|
||||
run: |
|
||||
@@ -110,19 +116,30 @@ jobs:
|
||||
|
||||
ubuntu-26-zdnn-s390x:
|
||||
name: ubuntu-26-zdnn-s390x
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
runs-on: ubuntu-24.04-s390x
|
||||
container: ubuntu:26.04 # required to get GCC 15.1 and binutils 2.44
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential cmake git ca-certificates \
|
||||
libssl-dev libzdnn-dev
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build Dependencies
|
||||
id: build_depends
|
||||
- name: Toolchain workaround (GCC 15)
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential libssl-dev libzdnn-dev
|
||||
apt-get install -y gcc-15 g++-15
|
||||
echo "CC=gcc-15" >> "$GITHUB_ENV"
|
||||
echo "CXX=g++-15" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build with zDNN Backend
|
||||
id: cmake_build
|
||||
|
||||
@@ -172,8 +172,6 @@ jobs:
|
||||
- [Ubuntu x64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.tar.gz)
|
||||
- [Ubuntu arm64 (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-arm64.tar.gz)
|
||||
- [Ubuntu s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-s390x.tar.gz)
|
||||
- [Ubuntu 26.04 s390x (CPU)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-26.04-s390x.tar.gz)
|
||||
- [Ubuntu 26.04 s390x (zDNN)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-zdnn-s390x.tar.gz)
|
||||
- [Ubuntu x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-x64.tar.gz)
|
||||
- [Ubuntu arm64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-vulkan-arm64.tar.gz)
|
||||
- [Ubuntu x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz) - [CUDA 12.8 libraries](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-${{ steps.tag.outputs.name }}-bin-ubuntu-cuda-12.8-x64.tar.gz)
|
||||
|
||||
@@ -180,8 +180,6 @@ jobs:
|
||||
os: ubuntu-24.04-arm
|
||||
- build: 's390x'
|
||||
os: ubuntu-24.04-s390x
|
||||
- build: '26.04-s390x'
|
||||
os: ubuntu-26.04-s390x
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
@@ -202,7 +200,7 @@ jobs:
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libssl-dev
|
||||
sudo apt-get install build-essential libssl-dev
|
||||
|
||||
- name: Toolchain workaround (GCC 14)
|
||||
if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
|
||||
@@ -212,7 +210,7 @@ jobs:
|
||||
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
||||
|
||||
- name: ccache
|
||||
if: ${{ !contains(matrix.os, 's390x') }}
|
||||
if: ${{ matrix.build != 's390x' }}
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
key: release-${{ matrix.os }}-cpu
|
||||
@@ -244,7 +242,7 @@ jobs:
|
||||
archive: false
|
||||
|
||||
- name: ccache-clear
|
||||
if: ${{ !contains(matrix.os, 's390x') }}
|
||||
if: ${{ matrix.build != 's390x' }}
|
||||
uses: ./.github/actions/ccache-clear
|
||||
with:
|
||||
key: release-${{ matrix.os }}-cpu
|
||||
@@ -1628,60 +1626,6 @@ jobs:
|
||||
with:
|
||||
key: release-ubuntu-24.04-rocm-${{ matrix.ROCM_VERSION }}-${{ matrix.build }}
|
||||
|
||||
ubuntu-26-zdnn:
|
||||
needs: [check-release, ui-build]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
runs-on: ubuntu-26.04-s390x
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download UI build
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential libssl-dev libzdnn-dev
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
cmake -B build \
|
||||
-DCMAKE_INSTALL_RPATH='$ORIGIN' \
|
||||
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
|
||||
-DGGML_BACKEND_DL=ON \
|
||||
-DGGML_NATIVE=OFF \
|
||||
-DGGML_VXE=ON \
|
||||
-DGGML_ZDNN=ON \
|
||||
-DCMAKE_C_FLAGS="-march=arch15" \
|
||||
-DCMAKE_CXX_FLAGS="-march=arch15" \
|
||||
-DLLAMA_FATAL_WARNINGS=ON \
|
||||
${{ env.CMAKE_ARGS }}
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Pack artifacts
|
||||
id: pack_artifacts
|
||||
run: |
|
||||
cp LICENSE ./build/bin/
|
||||
tar -czvf llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz --transform "s,^\.,llama-${{ needs.check-release.outputs.tag_name }}," -C ./build/bin .
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
path: llama-${{ needs.check-release.outputs.tag_name }}-bin-ubuntu-zdnn-s390x.tar.gz
|
||||
archive: false
|
||||
|
||||
ios-xcode:
|
||||
needs: [check-release]
|
||||
if: ${{ needs.check-release.outputs.should_release == 'true' }}
|
||||
|
||||
+1
-4
@@ -4,7 +4,7 @@ include(CheckIncludeFileCXX)
|
||||
|
||||
### llama.cpp version
|
||||
set(LLAMA_VERSION_MAJOR 0)
|
||||
set(LLAMA_VERSION_MINOR 8)
|
||||
set(LLAMA_VERSION_MINOR 7)
|
||||
set(LLAMA_VERSION_PATCH 0)
|
||||
set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}")
|
||||
|
||||
@@ -150,7 +150,6 @@ option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured
|
||||
# Required for relocatable CMake package
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/win-app.cmake)
|
||||
|
||||
if (NOT DEFINED LLAMA_BUILD_NUMBER)
|
||||
set(LLAMA_BUILD_NUMBER ${BUILD_NUMBER})
|
||||
@@ -270,8 +269,6 @@ if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS))
|
||||
add_subdirectory(tools/mtmd)
|
||||
endif()
|
||||
|
||||
llama_apply_app_manifests()
|
||||
|
||||
#
|
||||
# install
|
||||
#
|
||||
|
||||
+29
-22
@@ -2,29 +2,20 @@
|
||||
# multiple collaborators per item can be specified
|
||||
#
|
||||
# ggml-org/ci : CISC, danbev, ggerganov, netrunnereve, ngxson, taronaeo
|
||||
# ggml-org/ggml-base : aendk, fairydreaming, ggerganov
|
||||
# ggml-org/ggml-cann : hipudding
|
||||
# ggml-org/ggml-cpu : angt, chaxu01, ggerganov, max-krasnyansky, taronaeo
|
||||
# ggml-org/ggml-cuda : am17an, gaugarg-nv, IMbackK, JohannesGaessler, ORippler, thelittlefireman, ynankani
|
||||
# ggml-org/ggml-et : marty1885
|
||||
# ggml-org/ggml-cuda : JohannesGaessler, am17an, IMbackK, ORippler
|
||||
# ggml-org/ggml-hexagon : lhez, max-krasnyansky
|
||||
# ggml-org/ggml-hip : IMbackK, thelittlefireman
|
||||
# ggml-org/ggml-metal : forforever73, ggerganov, jhen0409, masterFoad
|
||||
# ggml-org/ggml-metal : ggerganov
|
||||
# ggml-org/ggml-opencl : lhez, max-krasnyansky
|
||||
# ggml-org/ggml-openvino : ravi9, wine99
|
||||
# ggml-org/ggml-riscv : xctan
|
||||
# ggml-org/ggml-rpc : am17an, rgerganov
|
||||
# ggml-org/ggml-sycl : arthw, Titaniumtown
|
||||
# ggml-org/ggml-rpc : rgerganov
|
||||
# ggml-org/ggml-sycl : arthw
|
||||
# ggml-org/ggml-vulkan : 0cc4m, jeffbolznv
|
||||
# ggml-org/ggml-webgpu : reeselevine, yomaytk
|
||||
# ggml-org/ggml-zdnn : AlekseiNikiforovIBM, Andreas-Krebbel, taronaeo
|
||||
# ggml-org/llama-base : am17an, CISC, ggerganov, ngxson, sanmai, ServeurpersoCom
|
||||
# ggml-org/llama-common : aldehir, angt, bartowski1182, danbev, ggerganov, ngxson, nikwen, pwilkin, ruixiang63, sanmai
|
||||
# ggml-org/llama-examples : ggerganov
|
||||
# ggml-org/llama-models : CISC, danbev, ggerganov, ngxson
|
||||
# ggml-org/ggml-zdnn : taronaeo
|
||||
# ggml-org/llama-common : ggerganov, aldehir, angt, danbev, ngxson, pwilkin
|
||||
# ggml-org/llama-mtmd : ngxson
|
||||
# ggml-org/llama-server : allozaur, angt, ggerganov, ngxson, ServeurpersoCom
|
||||
# ggml-org/llama-ui : allozaur
|
||||
# ggml-org/llama-server : ggerganov, ngxson, allozaur, angt, ServeurpersoCom
|
||||
# ggml-org/llama-ui : allozaur
|
||||
|
||||
/.devops/*.Dockerfile @ngxson
|
||||
/.github/actions/ @ggml-org/ci
|
||||
@@ -38,12 +29,28 @@
|
||||
/conversion/ @CISC
|
||||
/convert_*.py @CISC
|
||||
/docs/backend/snapdragon/ @ggml-org/ggml-hexagon
|
||||
/examples/ @ggml-org/llama-examples
|
||||
/examples/batched.swift/ @ggerganov
|
||||
/examples/batched/ @ggerganov
|
||||
/examples/convert-llama2c-to-ggml/ @ggerganov
|
||||
/examples/debug/ @danbev @pwilkin
|
||||
/examples/deprecation-warning/ @ggerganov
|
||||
/examples/diffusion/ @am17an
|
||||
/examples/embedding/ @ggerganov
|
||||
/examples/eval-callback/ @ggerganov
|
||||
/examples/export-docs/ @ggerganov
|
||||
/examples/gen-docs/ @ggerganov
|
||||
/examples/gguf/ @ggerganov
|
||||
/examples/llama.android/ @ggerganov @hanyin-arm @naco-siren
|
||||
/examples/llama.swiftui/ @ggerganov
|
||||
/examples/llama.vim @ggerganov
|
||||
/examples/lookahead/ @ggerganov
|
||||
/examples/lookup/ @JohannesGaessler
|
||||
/examples/model-conversion/ @danbev
|
||||
/examples/parallel/ @ggerganov
|
||||
/examples/passkey/ @ggerganov
|
||||
/examples/retrieval/ @ggerganov
|
||||
/examples/speculative-simple/ @ggerganov
|
||||
/examples/speculative/ @ggerganov
|
||||
/ggml/cmake/ @ggerganov
|
||||
/ggml/include/ @ggerganov
|
||||
/ggml/src/ggml-backend-meta.cpp @JohannesGaessler
|
||||
@@ -55,7 +62,7 @@
|
||||
/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda
|
||||
/ggml/src/ggml-cuda/vendors/hip.h @IMbackK
|
||||
/ggml/src/ggml-hexagon/ @ggml-org/ggml-hexagon
|
||||
/ggml/src/ggml-hip/ @ggml-org/ggml-hip
|
||||
/ggml/src/ggml-hip/ @IMbackK
|
||||
/ggml/src/ggml-et/ @marty1885
|
||||
/ggml/src/ggml-impl.h @ggerganov
|
||||
/ggml/src/ggml-metal/ @ggml-org/ggml-metal
|
||||
@@ -69,7 +76,7 @@
|
||||
/ggml/src/ggml-virtgpu/ @kpouget
|
||||
/ggml/src/ggml-vulkan/ @ggml-org/ggml-vulkan
|
||||
/ggml/src/ggml-webgpu/ @ggml-org/ggml-webgpu
|
||||
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn
|
||||
/ggml/src/ggml-zdnn/ @ggml-org/ggml-zdnn @Andreas-Krebbel @AlekseiNikiforovIBM
|
||||
/ggml/src/ggml-zendnn/ @avinashcpandey @Jiten1parmar
|
||||
/ggml/src/ggml.c @ggerganov
|
||||
/ggml/src/ggml.cpp @ggerganov
|
||||
@@ -80,14 +87,14 @@
|
||||
/scripts/get* @ggerganov
|
||||
/scripts/sync* @ggerganov
|
||||
/scripts/snapdragon/ @ggml-org/ggml-hexagon
|
||||
/src/ @ggml-org/llama-base
|
||||
/src/ @ggerganov
|
||||
/src/llama-adapter.* @CISC
|
||||
/src/llama-arch.* @CISC
|
||||
/src/llama-chat.* @ngxson
|
||||
/src/llama-graph.* @CISC
|
||||
/src/llama-model.* @CISC
|
||||
/src/llama-vocab.* @CISC
|
||||
/src/models/ @ggml-org/llama-models
|
||||
/src/models/ @CISC
|
||||
/tests/ @ggerganov
|
||||
/tests/test-chat.* @pwilkin
|
||||
/tests/test-server-component.cpp @ggml-org/llama-server
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
set(LLAMA_WIN_APP_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/../common/win-app.manifest")
|
||||
|
||||
if(WIN32)
|
||||
if(MSVC OR CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
|
||||
set(LLAMA_WIN_APP_MSVC_ABI TRUE)
|
||||
else()
|
||||
set(LLAMA_WIN_APP_MSVC_ABI FALSE)
|
||||
endif()
|
||||
|
||||
if(NOT LLAMA_WIN_APP_MSVC_ABI)
|
||||
if(CMAKE_VERSION VERSION_LESS 3.21)
|
||||
message(FATAL_ERROR "CMake 3.21 or newer is required to build for Windows with the GNU ABI")
|
||||
endif()
|
||||
enable_language(RC)
|
||||
set(LLAMA_WIN_APP_MANIFEST_RC "${CMAKE_CURRENT_BINARY_DIR}/win-app-manifest.rc")
|
||||
configure_file("${CMAKE_CURRENT_LIST_DIR}/win-app.rc.in" "${LLAMA_WIN_APP_MANIFEST_RC}" @ONLY)
|
||||
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
function(llama_add_app_manifest target)
|
||||
if(NOT WIN32)
|
||||
return()
|
||||
endif()
|
||||
get_target_property(_aliased ${target} ALIASED_TARGET)
|
||||
if(_aliased)
|
||||
return()
|
||||
endif()
|
||||
get_target_property(_type ${target} TYPE)
|
||||
if(NOT _type STREQUAL "EXECUTABLE")
|
||||
return()
|
||||
endif()
|
||||
if(LLAMA_WIN_APP_MSVC_ABI)
|
||||
target_sources(${target} PRIVATE "${LLAMA_WIN_APP_MANIFEST}")
|
||||
else()
|
||||
if(NOT TARGET llama-win-app-manifest-res)
|
||||
add_library(llama-win-app-manifest-res OBJECT ${LLAMA_WIN_APP_MANIFEST_RC})
|
||||
set_source_files_properties(${LLAMA_WIN_APP_MANIFEST_RC} PROPERTIES OBJECT_DEPENDS "${LLAMA_WIN_APP_MANIFEST}")
|
||||
endif()
|
||||
target_link_libraries(${target} PRIVATE "$<TARGET_OBJECTS:llama-win-app-manifest-res>")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(llama_apply_app_manifests_dir dir)
|
||||
get_property(_targets DIRECTORY "${dir}" PROPERTY BUILDSYSTEM_TARGETS)
|
||||
foreach(_target IN LISTS _targets)
|
||||
llama_add_app_manifest(${_target})
|
||||
endforeach()
|
||||
get_property(_subdirs DIRECTORY "${dir}" PROPERTY SUBDIRECTORIES)
|
||||
foreach(_subdir IN LISTS _subdirs)
|
||||
llama_apply_app_manifests_dir("${_subdir}")
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
function(llama_apply_app_manifests)
|
||||
if(NOT WIN32)
|
||||
return()
|
||||
endif()
|
||||
get_property(_subdirs DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY SUBDIRECTORIES)
|
||||
foreach(_dir IN LISTS _subdirs)
|
||||
get_filename_component(_name "${_dir}" NAME)
|
||||
if(_name STREQUAL "app" OR _name STREQUAL "examples" OR _name STREQUAL "pocs" OR _name STREQUAL "tests" OR _name STREQUAL "tools")
|
||||
llama_apply_app_manifests_dir("${_dir}")
|
||||
endif()
|
||||
endforeach()
|
||||
endfunction()
|
||||
@@ -1,3 +0,0 @@
|
||||
// generated from cmake/win-app.rc.in - do not edit
|
||||
// RT_MANIFEST (24), ID 1: the application manifest shared by all llama.cpp executables
|
||||
1 24 "@LLAMA_WIN_APP_MANIFEST@"
|
||||
@@ -1803,9 +1803,6 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
|
||||
//LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());
|
||||
|
||||
common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;
|
||||
if (!is_partial) {
|
||||
flags |= COMMON_PEG_PARSE_FLAG_COMPLETE;
|
||||
}
|
||||
if (params.debug) {
|
||||
flags |= COMMON_PEG_PARSE_FLAG_DEBUG;
|
||||
}
|
||||
|
||||
+6
-27
@@ -219,23 +219,6 @@ int32_t common_cpu_get_num_math() {
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(__riscv) && defined(__linux__)
|
||||
std::ifstream cpuinfo("/proc/cpuinfo");
|
||||
std::string line;
|
||||
int32_t n_a100 = 0;
|
||||
while (std::getline(cpuinfo, line)) {
|
||||
std::istringstream fields(line);
|
||||
std::string key;
|
||||
std::string separator;
|
||||
uint64_t marchid;
|
||||
if (fields >> key >> separator >> std::hex >> marchid &&
|
||||
key == "marchid" && separator == ":" && marchid == UINT64_C(0x8000000041000002)) {
|
||||
++n_a100;
|
||||
}
|
||||
}
|
||||
if (n_a100 > 0) {
|
||||
return n_a100;
|
||||
}
|
||||
#elif defined(__powerpc64__) || defined(__powerpc__)
|
||||
int32_t smt_factor = 1;
|
||||
int phy_cpus = common_cpu_get_num_physical_cores();
|
||||
@@ -1490,7 +1473,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
|
||||
if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
|
||||
if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);
|
||||
|
||||
const auto cvec = common_control_vector_load(params.control_vectors, llama_model_n_layer(model));
|
||||
const auto cvec = common_control_vector_load(params.control_vectors);
|
||||
if (cvec.n_embd == -1) {
|
||||
return res;
|
||||
}
|
||||
@@ -2012,7 +1995,7 @@ float common_embd_similarity_cos(const float * embd1, const float * embd2, int n
|
||||
// Control vector utils
|
||||
//
|
||||
|
||||
static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info, int32_t n_layer) {
|
||||
static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info) {
|
||||
common_control_vector_data result = { -1, {} };
|
||||
|
||||
ggml_context * ctx = nullptr;
|
||||
@@ -2053,10 +2036,6 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str());
|
||||
result.n_embd = -1;
|
||||
break;
|
||||
} else if (layer_idx > n_layer) {
|
||||
COM_ERR("direction tensor layer index %d exceeds the model layer count %d in %s\n", layer_idx, n_layer, load_info.fname.c_str());
|
||||
result.n_embd = -1;
|
||||
break;
|
||||
}
|
||||
|
||||
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());
|
||||
@@ -2080,10 +2059,10 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
}
|
||||
|
||||
// extend if necessary - do not store data for layer 0 (it's not used)
|
||||
result.data.resize(std::max(result.data.size(), (size_t) result.n_embd * layer_idx), 0.0f);
|
||||
result.data.resize(std::max(result.data.size(), static_cast<size_t>(result.n_embd * layer_idx)), 0.0f);
|
||||
|
||||
const float * src = (const float *) tensor->data;
|
||||
float * dst = result.data.data() + (size_t) result.n_embd * (layer_idx - 1); // layer 1 at [0]
|
||||
float * dst = result.data.data() + result.n_embd * (layer_idx - 1); // layer 1 at [0]
|
||||
for (int j = 0; j < result.n_embd; j++) {
|
||||
dst[j] += src[j] * load_info.strength; // allows multiple directions for same layer in same file
|
||||
}
|
||||
@@ -2101,11 +2080,11 @@ static common_control_vector_data common_control_vector_load_one(const common_co
|
||||
return result;
|
||||
}
|
||||
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer) {
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos) {
|
||||
common_control_vector_data result = { -1, {} };
|
||||
|
||||
for (const auto & info : load_infos) {
|
||||
auto cur = common_control_vector_load_one(info, n_layer);
|
||||
auto cur = common_control_vector_load_one(info);
|
||||
|
||||
if (cur.n_embd == -1) {
|
||||
result.n_embd = -1;
|
||||
|
||||
+1
-2
@@ -1237,9 +1237,8 @@ struct common_control_vector_load_info {
|
||||
};
|
||||
|
||||
// Load control vectors, scale each by strength, and add them together.
|
||||
// Direction layer indices must be in [1, n_layer].
|
||||
// On error, returns {-1, empty}
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer);
|
||||
common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos);
|
||||
|
||||
//
|
||||
// Split utils
|
||||
|
||||
@@ -76,10 +76,9 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (supports_reasoning && extract_reasoning) {
|
||||
// The template strips the newline before </think>, so leave it out of the reasoning
|
||||
reasoning = p.optional("<think>" + p.space() +
|
||||
p.reasoning(p.until_one_of({ "\n</think>", "</think>", "<tool_call>" })) +
|
||||
(p.literal("\n</think>") | p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
|
||||
p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
|
||||
(p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
|
||||
}
|
||||
|
||||
// Response format parser
|
||||
@@ -167,15 +166,8 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
|
||||
auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
|
||||
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
|
||||
|
||||
// The templates separate content from the tool calls with one or two newlines,
|
||||
// stop the content before them
|
||||
std::vector<std::string> content_ends;
|
||||
for (const auto & start : tool_call_starts) {
|
||||
content_ends.insert(content_ends.end(), { "\n\n" + start, "\n" + start, start });
|
||||
}
|
||||
|
||||
return generation_prompt +
|
||||
(reasoning << p.content(p.until_one_of(content_ends)) << tool_calls);
|
||||
(reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
|
||||
}
|
||||
|
||||
// Content only parser
|
||||
|
||||
@@ -721,9 +721,8 @@ struct parser_executor {
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
|
||||
}
|
||||
|
||||
if (match == common_trie::PARTIAL_MATCH && !ctx.is_complete()) {
|
||||
// Found a partial match extending to end of input, return everything before it.
|
||||
// On a complete input the delimiter can no longer form, so the bytes belong to the text
|
||||
if (match == common_trie::PARTIAL_MATCH) {
|
||||
// Found a partial match extending to end of input, return everything before it
|
||||
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
|
||||
}
|
||||
|
||||
|
||||
+3
-5
@@ -162,10 +162,9 @@ struct common_peg_parse_result {
|
||||
};
|
||||
|
||||
enum common_peg_parse_flags {
|
||||
COMMON_PEG_PARSE_FLAG_NONE = 0,
|
||||
COMMON_PEG_PARSE_FLAG_LENIENT = 1 << 0,
|
||||
COMMON_PEG_PARSE_FLAG_DEBUG = 1 << 1,
|
||||
COMMON_PEG_PARSE_FLAG_COMPLETE = 1 << 2,
|
||||
COMMON_PEG_PARSE_FLAG_NONE = 0,
|
||||
COMMON_PEG_PARSE_FLAG_LENIENT = 1 << 0,
|
||||
COMMON_PEG_PARSE_FLAG_DEBUG = 1 << 1,
|
||||
};
|
||||
|
||||
inline common_peg_parse_flags operator|(common_peg_parse_flags a, common_peg_parse_flags b) {
|
||||
@@ -210,7 +209,6 @@ struct common_peg_parse_context {
|
||||
|
||||
bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }
|
||||
bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }
|
||||
bool is_complete() const { return flags & COMMON_PEG_PARSE_FLAG_COMPLETE; }
|
||||
};
|
||||
|
||||
class common_peg_arena;
|
||||
|
||||
@@ -136,8 +136,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to
|
||||
// track the end sequence within forced_tokens so it is also reported on DONE
|
||||
const int32_t match = ctx->end_matcher.advance(token);
|
||||
ctx->force_pos++;
|
||||
// an end tag that is already in the prompt (e.g. "<think></think>") also ends forcing
|
||||
if (match >= 0 || ctx->force_pos >= ctx->forced_tokens.size()) {
|
||||
if (ctx->force_pos >= ctx->forced_tokens.size()) {
|
||||
ctx->state = REASONING_BUDGET_DONE;
|
||||
ctx->end_match = match;
|
||||
COM_TRC("%s", "forced sequence complete, done\n");
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
|
||||
<application xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
<windowsSettings>
|
||||
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
|
||||
<longPathAware xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">true</longPathAware>
|
||||
</windowsSettings>
|
||||
</application>
|
||||
</assembly>
|
||||
@@ -176,7 +176,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Lfm3ForCausalLM": "lfm3",
|
||||
"Lfm3OmniForConditionalGeneration": "lfm3",
|
||||
"Lfm3VlForConditionalGeneration": "lfm3",
|
||||
"LimiteForCausalLM": "limite",
|
||||
"Llama4ForCausalLM": "llama",
|
||||
"Llama4ForConditionalGeneration": "llama",
|
||||
"LlamaBidirectionalModel": "llama",
|
||||
@@ -213,7 +212,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Ministral3Model": "mistral3",
|
||||
"Mistral3ForConditionalGeneration": "mistral3",
|
||||
"Mistral3Model": "mistral3",
|
||||
"MistralLarge4ForConditionalGeneration": "mistral4",
|
||||
"VoxtralRealtimeForConditionalGeneration": "mistral3",
|
||||
"MistralForCausalLM": "llama",
|
||||
"MixtralForCausalLM": "llama",
|
||||
|
||||
+66
-301
@@ -16,7 +16,6 @@ from pathlib import Path
|
||||
from hashlib import sha256
|
||||
from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Literal, Sequence, TypeVar, cast
|
||||
from itertools import chain
|
||||
from math import prod
|
||||
from transformers import AutoConfig
|
||||
|
||||
import numpy as np
|
||||
@@ -156,7 +155,6 @@ class ModelBase:
|
||||
self.fuse_gate_up_exps = fuse_gate_up_exps
|
||||
self._gate_exp_buffer: dict[int, Tensor] = {}
|
||||
self._up_exp_buffer: dict[int, Tensor] = {}
|
||||
self._unfusable_gate_up_layers: set[int] = set()
|
||||
self.fuse_qkv = fuse_qkv
|
||||
self._q_buffer: dict[int, Tensor] = {}
|
||||
self._k_buffer: dict[int, Tensor] = {}
|
||||
@@ -177,9 +175,7 @@ class ModelBase:
|
||||
self._nvfp4_layer_algo: dict[str, str | None] = {} # per-layer quant_algo, keyed by HF module path
|
||||
self._prec_a4: dict[str, bool] = {} # gguf tensor name -> can use 4-bit (A4) activations
|
||||
self._fp8_as_q8 = fp8_as_q8
|
||||
self._preserve_fp8 = ftype == gguf.LlamaFileType.GUESSED and not fp8_as_q8
|
||||
self._fp8_dequantized: set[str] = set()
|
||||
self._fp8_e4m3_preserved: set[str] = set()
|
||||
|
||||
# Apply heuristics to figure out typical tensor encoding based on first tensor's dtype
|
||||
# NOTE: can't use field "torch_dtype" in config.json, because some finetunes lie.
|
||||
@@ -330,7 +326,7 @@ class ModelBase:
|
||||
|
||||
def dequant_model(self):
|
||||
# If all quantized tensors were already handled (e.g. pure NVFP4), skip
|
||||
if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv", ".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")) for k in self.model_tensors):
|
||||
if self._is_nvfp4 and not any(k.endswith((".weight_scale", ".weight_scale_inv")) for k in self.model_tensors):
|
||||
return
|
||||
|
||||
tensors_to_remove: list[str] = []
|
||||
@@ -354,12 +350,6 @@ class ModelBase:
|
||||
def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor:
|
||||
scale = scale.float()
|
||||
|
||||
# ModelOpt block scales are pre-shaped for broadcasting, e.g. [out/128, 1, in/128, 1]
|
||||
if block_size is None and scale.ndim == 2 * weight.ndim and all(n == 1 for n in scale.shape[1::2]):
|
||||
assert all(w % n == 0 for w, n in zip(weight.shape, scale.shape[0::2]))
|
||||
block_size = [w // n for w, n in zip(weight.shape, scale.shape[0::2])]
|
||||
scale = scale.reshape(tuple(scale.shape[0::2]))
|
||||
|
||||
if block_size is not None:
|
||||
dim_offset = scale.ndim - len(block_size)
|
||||
for i, size in enumerate(block_size):
|
||||
@@ -450,7 +440,7 @@ class ModelBase:
|
||||
|
||||
return (unpacked * scale.unsqueeze(-1).float()).reshape(shape)
|
||||
|
||||
def dequant_fp8(fp8_configs: list[dict[str, Any]] | None = None) -> None:
|
||||
def dequant_fp8() -> None:
|
||||
for name in self.model_tensors.keys():
|
||||
if name.endswith(".weight_scale"):
|
||||
weight_name = name.removesuffix("_scale")
|
||||
@@ -459,29 +449,14 @@ class ModelBase:
|
||||
continue
|
||||
w = self.model_tensors[weight_name]
|
||||
s = self.model_tensors[name]
|
||||
block_size = None
|
||||
is_fp8_weight = False
|
||||
if self._fp8_as_q8 or fp8_configs is not None:
|
||||
weight = w()
|
||||
is_fp8_weight = weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
|
||||
if fp8_configs is not None:
|
||||
if not is_fp8_weight:
|
||||
raise ValueError(f"Expected remaining weight {weight_name!r} to be FP8, got {weight.dtype}")
|
||||
scale = s()
|
||||
is_channel_scale = scale.ndim >= 1 and scale.ndim <= weight.ndim and scale.shape[0] == weight.shape[0] and all(size == 1 for size in scale.shape[1:])
|
||||
if prod(scale.shape) != 1 and not is_channel_scale:
|
||||
block_configs = [c for c in fp8_configs if c.get("strategy") == "block"]
|
||||
if not block_configs or any(not c.get("block_structure") or c.get("group_size") is not None for c in block_configs):
|
||||
raise ValueError(f"FP8 weight {weight_name!r} with scale shape {list(scale.shape)} requires a block layout")
|
||||
block_sizes = {tuple(c["block_structure"]) for c in block_configs}
|
||||
if len(block_sizes) != 1:
|
||||
raise ValueError(f"FP8 weight {weight_name!r} has ambiguous block layouts")
|
||||
block_size = next(iter(block_sizes))
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
|
||||
if self._fp8_as_q8:
|
||||
is_fp8_weight = w().dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), None)
|
||||
tensors_to_remove.append(name)
|
||||
if self._fp8_as_q8 and is_fp8_weight:
|
||||
if is_fp8_weight:
|
||||
self._fp8_dequantized.add(weight_name)
|
||||
if name.endswith((".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")):
|
||||
if name.endswith((".input_scale", ".k_scale", ".v_scale")):
|
||||
tensors_to_remove.append(name)
|
||||
|
||||
if quant_method == "bitnet":
|
||||
@@ -546,37 +521,29 @@ class ModelBase:
|
||||
nvfp4_compressed_tensors = self._is_nvfp4_compressed_tensors(quant_method, quant_format, groups)
|
||||
|
||||
if nvfp4_compressed_tensors:
|
||||
fp8_configs = [
|
||||
g["weights"] for g in groups.values()
|
||||
if isinstance(g, dict) and g.get("format") == "float-quantized"
|
||||
and g.get("weights", {}).get("type") == "float"
|
||||
and g.get("weights", {}).get("num_bits") == 8
|
||||
]
|
||||
dequant_fp8(fp8_configs)
|
||||
dequant_fp8()
|
||||
elif len(groups) > 1:
|
||||
raise NotImplementedError("Can't handle multiple config groups for compressed-tensors yet")
|
||||
elif quant_format == "float-quantized" or quant_format == "int-quantized" or quant_format == "naive-quantized":
|
||||
weight_config = tuple(groups.values())[0]["weights"]
|
||||
block_size = weight_config.get("block_structure", None)
|
||||
strategy = weight_config.get("strategy")
|
||||
assert strategy == "channel" or strategy == "block"
|
||||
assert weight_config.get("group_size") is None # didn't find a model using this yet
|
||||
is_fp8 = (
|
||||
quant_format == "float-quantized"
|
||||
and weight_config.get("type") == "float"
|
||||
and weight_config.get("num_bits") == 8
|
||||
)
|
||||
assert strategy in ("channel", "block") or (is_fp8 and strategy == "tensor")
|
||||
assert weight_config.get("group_size") is None # didn't find a model using this yet
|
||||
for name in self.model_tensors.keys():
|
||||
if name.endswith(".weight_scale"):
|
||||
weight_name = name.removesuffix("_scale")
|
||||
w = self.model_tensors[weight_name]
|
||||
s = self.model_tensors[name]
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), block_size)
|
||||
tensors_to_remove.append(name)
|
||||
if self._fp8_as_q8 and is_fp8:
|
||||
self._fp8_dequantized.add(weight_name)
|
||||
if is_fp8 and name.endswith((".input_scale", ".activation_scale", "_activation_scale", ".k_scale", ".v_scale")):
|
||||
tensors_to_remove.append(name)
|
||||
elif quant_format == "pack-quantized":
|
||||
weight_config = tuple(groups.values())[0]["weights"]
|
||||
assert weight_config.get("strategy") == "group"
|
||||
@@ -614,148 +581,6 @@ class ModelBase:
|
||||
for name, value in new_tensors.items():
|
||||
self.model_tensors[name] = value
|
||||
|
||||
def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
|
||||
new_name = self.tensor_map.get_name(name, try_suffixes=(".weight",))
|
||||
return (new_name,) if new_name is not None else ()
|
||||
|
||||
def _prepare_fp8_e4m3_tensors(self):
|
||||
if not self._preserve_fp8:
|
||||
return
|
||||
|
||||
scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
|
||||
input_scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
|
||||
consumed: list[str] = []
|
||||
candidates: dict[str, tuple[str, Tensor, tuple[str, ...], torch.Size]] = {}
|
||||
weight_dtypes: dict[str, torch.dtype] = {}
|
||||
scaled_weights: set[str] = set()
|
||||
expert_pattern = re.compile(r"(\.[^.]*experts)\.(\d+)\.")
|
||||
scale_types = {
|
||||
gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V,
|
||||
gguf.MODEL_TENSOR.ATTN_OUT, gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE,
|
||||
gguf.MODEL_TENSOR.FFN_GATE, gguf.MODEL_TENSOR.FFN_UP, gguf.MODEL_TENSOR.FFN_DOWN,
|
||||
gguf.MODEL_TENSOR.FFN_GATE_SHEXP, gguf.MODEL_TENSOR.FFN_UP_SHEXP, gguf.MODEL_TENSOR.FFN_DOWN_SHEXP,
|
||||
gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
|
||||
gguf.MODEL_TENSOR.SSM_IN, gguf.MODEL_TENSOR.SSM_OUT, gguf.MODEL_TENSOR.SSM_ALPHA, gguf.MODEL_TENSOR.SSM_BETA,
|
||||
gguf.MODEL_TENSOR.NEXTN_EH_PROJ, gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, gguf.MODEL_TENSOR.OUTPUT,
|
||||
}
|
||||
|
||||
for scale_name in list(self.model_tensors.keys()):
|
||||
weight_name = None
|
||||
if scale_name.endswith(".weight_scale"):
|
||||
weight_name = scale_name.removesuffix("_scale")
|
||||
elif scale_name.endswith("_scale_inv"):
|
||||
weight_name = scale_name.removesuffix("_scale_inv")
|
||||
elif scale_name.endswith(".qscale_weight"):
|
||||
weight_name = scale_name.removesuffix("qscale_weight") + "weight"
|
||||
|
||||
if weight_name is None or weight_name not in self.model_tensors:
|
||||
continue
|
||||
|
||||
scaled_weights.add(weight_name)
|
||||
weight = self.model_tensors[weight_name]()
|
||||
weight_dtypes[weight_name] = weight.dtype
|
||||
if weight.dtype != torch.float8_e4m3fn or weight.ndim < 2:
|
||||
continue
|
||||
|
||||
scale = self.model_tensors[scale_name]()
|
||||
if prod(scale.shape) != 1:
|
||||
continue
|
||||
|
||||
merged_name = expert_pattern.sub(r"\1.", weight_name)
|
||||
new_names = self._map_fp8_weight_names(merged_name)
|
||||
if not new_names:
|
||||
continue
|
||||
if any((mapped := self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",))) is None or mapped[0] not in scale_types for new_name in new_names):
|
||||
continue
|
||||
bid = next((int(part) for part in new_names[0].split(".") if part.isdecimal()), None)
|
||||
self._fp8_e4m3_preserved.add(weight_name)
|
||||
if any(new_name.endswith("_norm.weight") or self.tensor_force_quant(weight_name, new_name, bid, weight.ndim) != gguf.GGMLQuantizationType.F8_E4M3 for new_name in new_names):
|
||||
self._fp8_e4m3_preserved.discard(weight_name)
|
||||
continue
|
||||
candidates[weight_name] = (scale_name, scale, new_names, weight.shape)
|
||||
|
||||
expert_groups: dict[str, list[str]] = {}
|
||||
for name in self.model_tensors:
|
||||
if (match := expert_pattern.search(name)) and name.endswith(".weight"):
|
||||
expert_groups.setdefault(name[:match.end(1)], []).append(name)
|
||||
for names in expert_groups.values():
|
||||
for name in names:
|
||||
if name not in weight_dtypes:
|
||||
weight_dtypes[name] = self.model_tensors[name]().dtype
|
||||
dtypes = {weight_dtypes[name] for name in names}
|
||||
if torch.float8_e4m3fn not in dtypes:
|
||||
continue
|
||||
if dtypes == {torch.float8_e4m3fn} and not any(name in scaled_weights for name in names):
|
||||
continue
|
||||
# The converter merges expert projections together.
|
||||
if not all(name in candidates for name in names):
|
||||
for name in names:
|
||||
candidates.pop(name, None)
|
||||
self._fp8_e4m3_preserved.discard(name)
|
||||
if name not in scaled_weights:
|
||||
gen = self.model_tensors[name]
|
||||
self.model_tensors[name] = lambda gen=gen: gen().float()
|
||||
|
||||
for weight_name, (scale_name, scale, new_names, shape) in candidates.items():
|
||||
scale = LazyTorchTensor.to_eager(scale).float().flatten()
|
||||
|
||||
weight_prefix = weight_name.removesuffix(".weight")
|
||||
# Transformers fine-grained FP8 uses activation_scale while ModelOpt uses input_scale.
|
||||
# Ref: https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/mistral3.py#L357-L361
|
||||
input_scale_name = next((name for name in (
|
||||
weight_prefix + ".input_scale",
|
||||
weight_prefix + ".activation_scale",
|
||||
weight_prefix + "_activation_scale",
|
||||
weight_prefix + ".qscale_act",
|
||||
) if name in self.model_tensors), None)
|
||||
input_scale = None
|
||||
if input_scale_name is not None:
|
||||
input_scale = self.model_tensors[input_scale_name]()
|
||||
if prod(input_scale.shape) != 1:
|
||||
raise ValueError(f"FP8 input scale {input_scale_name!r} must be a scalar")
|
||||
input_scale = LazyTorchTensor.to_eager(input_scale).float().flatten()
|
||||
consumed.append(input_scale_name)
|
||||
|
||||
consumed.append(scale_name)
|
||||
|
||||
expert_match = expert_pattern.search(weight_name)
|
||||
if expert_match:
|
||||
expert_id = int(expert_match.group(2))
|
||||
for new_name in new_names:
|
||||
target_name = new_name.replace(".weight", ".scale")
|
||||
entries = scale_tensors.setdefault(target_name, [])
|
||||
assert isinstance(entries, list)
|
||||
cast(list[tuple[int, float]], entries).append((expert_id, float(scale[0])))
|
||||
if input_scale is not None:
|
||||
target_name = new_name.replace(".weight", ".input_scale")
|
||||
entries = input_scale_tensors.setdefault(target_name, [])
|
||||
assert isinstance(entries, list)
|
||||
cast(list[tuple[int, float]], entries).append((expert_id, float(input_scale[0])))
|
||||
else:
|
||||
for new_name in new_names:
|
||||
mapped = self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",)) if len(shape) == 3 else None
|
||||
is_packed_expert = mapped is not None and mapped[0] in (
|
||||
gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
|
||||
)
|
||||
# Packed expert weights need one scale value per expert.
|
||||
n_scales = shape[0] if is_packed_expert else 1
|
||||
scale_tensors[new_name.replace(".weight", ".scale")] = np.repeat(scale.numpy(), n_scales)
|
||||
if input_scale is not None:
|
||||
input_scale_tensors[new_name.replace(".weight", ".input_scale")] = np.repeat(input_scale.numpy(), n_scales)
|
||||
|
||||
for name in consumed:
|
||||
self.model_tensors.pop(name, None)
|
||||
|
||||
for name, values in chain(scale_tensors.items(), input_scale_tensors.items()):
|
||||
if isinstance(values, list):
|
||||
values.sort(key=lambda item: item[0])
|
||||
scale = np.array([item[1] for item in values], dtype=np.float32)
|
||||
else:
|
||||
scale = values.astype(np.float32)
|
||||
if not np.allclose(scale, 1.0, atol=1e-6):
|
||||
logger.info(f" + {name} (FP8 scale, shape [{scale.size}])")
|
||||
self.gguf_writer.add_tensor(name, scale)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
@@ -800,26 +625,6 @@ class ModelBase:
|
||||
raise ValueError(f"Can not map tensor {name!r}")
|
||||
return new_name
|
||||
|
||||
def prepare_gate_up_fusion(self) -> None:
|
||||
self._unfusable_gate_up_layers.clear()
|
||||
if not self.fuse_gate_up_exps or self._fp8_as_q8:
|
||||
return
|
||||
|
||||
for name, gen in self.model_tensors.items():
|
||||
merged_name = re.sub(r"(\.experts)\.\d+\.", r"\1.", name)
|
||||
mapped = self.tensor_map.get_type_and_name(merged_name, try_suffixes=(".weight",))
|
||||
if mapped is None:
|
||||
continue
|
||||
tensor_type, new_name = mapped
|
||||
if tensor_type not in (gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_GATE_UP_EXP) or not new_name.endswith(".weight"):
|
||||
continue
|
||||
if gen().dtype == torch.float8_e4m3fn:
|
||||
bid = next(int(part) for part in new_name.split(".") if part.isdecimal())
|
||||
self._unfusable_gate_up_layers.add(bid)
|
||||
|
||||
for bid in sorted(self._unfusable_gate_up_layers):
|
||||
logger.warning(f"Skipping --fuse-gate-up-exps for layer {bid}: fusion of preserved FP8 expert projections is not supported")
|
||||
|
||||
def prepare_qkv_fusion(self) -> None:
|
||||
self._fusable_qkv_weight_layers.clear()
|
||||
self._fusable_qkv_bias_layers.clear()
|
||||
@@ -833,7 +638,6 @@ class ModelBase:
|
||||
}
|
||||
weights: dict[int, set[gguf.MODEL_TENSOR]] = {}
|
||||
biases: dict[int, set[gguf.MODEL_TENSOR]] = {}
|
||||
fp8_layers: set[int] = set()
|
||||
|
||||
for name in self.model_tensors:
|
||||
mapped = self.tensor_map.get_type_and_name(name, try_suffixes=(".weight", ".bias"))
|
||||
@@ -848,19 +652,12 @@ class ModelBase:
|
||||
continue
|
||||
if new_name.endswith(".weight"):
|
||||
weights.setdefault(bid, set()).add(tensor_type)
|
||||
if not self._fp8_as_q8 and self.model_tensors[name]().dtype == torch.float8_e4m3fn:
|
||||
fp8_layers.add(bid)
|
||||
elif new_name.endswith(".bias"):
|
||||
biases.setdefault(bid, set()).add(tensor_type)
|
||||
|
||||
for bid, weight_types in weights.items():
|
||||
bias_types = biases.get(bid, set())
|
||||
if weight_types == qkv_types and (not bias_types or bias_types == qkv_types):
|
||||
# NVFP4 weights use separate reordering and repacking hooks and are exported before this fusion pass.
|
||||
# Separate FP8 scales cannot be represented by one fused QKV scale.
|
||||
if bid in fp8_layers:
|
||||
logger.warning(f"Skipping --fuse-qkv for layer {bid}: fusion of preserved FP8 projections is not supported")
|
||||
continue
|
||||
self._fusable_qkv_weight_layers.add(bid)
|
||||
if bias_types:
|
||||
self._fusable_qkv_bias_layers.add(bid)
|
||||
@@ -1049,8 +846,7 @@ class ModelBase:
|
||||
new_name = self.map_tensor_name(name)
|
||||
|
||||
# Handle gate/up expert tensor fusion if enabled
|
||||
# Preserved FP8 gate/up scales must stay with their separate projections.
|
||||
if self.fuse_gate_up_exps and bid is not None and bid not in self._unfusable_gate_up_layers:
|
||||
if self.fuse_gate_up_exps and bid is not None:
|
||||
if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_GATE_EXP, bid):
|
||||
self._gate_exp_buffer[bid] = data_torch
|
||||
elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_UP_EXP, bid):
|
||||
@@ -1107,46 +903,8 @@ class ModelBase:
|
||||
|
||||
return [(new_name, data_torch)]
|
||||
|
||||
def _tensor_requires_f32(self, new_name: str, bid: int | None, n_dims: int) -> bool:
|
||||
# Conditions should closely match those in llama_model_quantize_internal in llama.cpp.
|
||||
return n_dims <= 1 or new_name.endswith("_norm.weight") or (
|
||||
any(
|
||||
self.match_model_tensor_name(new_name, key, bid)
|
||||
for key in (
|
||||
gguf.MODEL_TENSOR.FFN_GATE_INP,
|
||||
gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
|
||||
gguf.MODEL_TENSOR.POS_EMBD,
|
||||
gguf.MODEL_TENSOR.TOKEN_TYPES,
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D,
|
||||
gguf.MODEL_TENSOR.SHORTCONV_CONV,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_FIRST,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_W1,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_W2,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,
|
||||
gguf.MODEL_TENSOR.POSNET_NORM1,
|
||||
gguf.MODEL_TENSOR.POSNET_NORM2,
|
||||
gguf.MODEL_TENSOR.V_ENC_EMBD_POS,
|
||||
gguf.MODEL_TENSOR.A_ENC_EMBD_POS,
|
||||
gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,
|
||||
gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,
|
||||
# Kimi KDA conv weights should be F32
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D_Q,
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D_K,
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D_V,
|
||||
# DSA indexer weights should be F32
|
||||
gguf.MODEL_TENSOR.INDEXER_PROJ,
|
||||
)
|
||||
)
|
||||
or new_name[-7:] not in (".weight", ".lora_a", ".lora_b")
|
||||
)
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
if name in self._fp8_e4m3_preserved and n_dims >= 2:
|
||||
if self._tensor_requires_f32(new_name, bid, n_dims):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return gguf.GGMLQuantizationType.F8_E4M3
|
||||
del new_name, bid # unused
|
||||
# Force FP8-original tensors to Q8_0 when requested; Q8_0 is faster than F16/BF16.
|
||||
if self._fp8_as_q8 and name in self._fp8_dequantized and n_dims >= 2:
|
||||
return gguf.GGMLQuantizationType.Q8_0
|
||||
@@ -1277,7 +1035,7 @@ class ModelBase:
|
||||
expert_scales: dict[tuple[int, str], list[tuple[int, float]]] = {}
|
||||
expert_input_scales: dict[tuple[int, str], list[tuple[int, float]]] = {}
|
||||
expert_shapes: dict[tuple[int, str], list[int]] = {}
|
||||
n_experts = self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True) or 0
|
||||
n_experts = self.find_hparam(["num_local_experts", "num_experts"], optional=True) or 0
|
||||
consumed: list[str] = []
|
||||
|
||||
for name in self.model_tensors.keys():
|
||||
@@ -1288,17 +1046,14 @@ class ModelBase:
|
||||
input_scale_name = name.replace(".weight", ".input_scale")
|
||||
if scale_name not in self.model_tensors:
|
||||
continue
|
||||
weight = self.model_tensors[name]()
|
||||
scale = self.model_tensors[scale_name]()
|
||||
# Force eager materialization of lazy tensors
|
||||
weight = LazyTorchTensor.to_eager(self.model_tensors[name]())
|
||||
scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]())
|
||||
|
||||
# Leave FP8 weights and their scales for _prepare_fp8_e4m3_tensors.
|
||||
if weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2) or scale.ndim < 2:
|
||||
# Skip non-NVFP4 tensors(e.g. 1D scale, or float8 weight)
|
||||
if scale.ndim < 2 or weight.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
|
||||
continue
|
||||
if weight.ndim != 2 or scale.shape[0] != weight.shape[0] or scale.shape[1] * 8 != weight.shape[1]:
|
||||
raise ValueError(f"NVFP4 weight {name!r} has incompatible shapes: weight {list(weight.shape)}, scale {list(scale.shape)}")
|
||||
|
||||
weight = LazyTorchTensor.to_eager(weight)
|
||||
scale = LazyTorchTensor.to_eager(scale)
|
||||
scale2 = LazyTorchTensor.to_eager(self.model_tensors.get(scale2_name, lambda: torch.tensor(1.0))())
|
||||
input_scale = LazyTorchTensor.to_eager(self.model_tensors.get(input_scale_name, lambda: torch.tensor(1.0))())
|
||||
|
||||
@@ -1454,10 +1209,8 @@ class ModelBase:
|
||||
self.model_tensors[input_scale_name] = inverse_scale(self.model_tensors.pop(name))
|
||||
self._generate_nvfp4_tensors()
|
||||
|
||||
self._prepare_fp8_e4m3_tensors()
|
||||
self.dequant_model()
|
||||
|
||||
self.prepare_gate_up_fusion()
|
||||
self.prepare_qkv_fusion()
|
||||
|
||||
# Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
|
||||
@@ -1471,21 +1224,10 @@ class ModelBase:
|
||||
if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
|
||||
continue
|
||||
|
||||
# FP8 transforms must only reorder values; norms and vectors use numeric F32 transforms.
|
||||
preserve_fp8_e4m3 = self._preserve_fp8 and data_torch.dtype == torch.float8_e4m3fn and data_torch.ndim >= 2 and not name.endswith(("norm.weight", "norm_f.weight"))
|
||||
if preserve_fp8_e4m3:
|
||||
for new_name in self._map_fp8_weight_names(name):
|
||||
mapped_bid = next((int(part) for part in new_name.split(".") if part.isdecimal()), None)
|
||||
if self._tensor_requires_f32(new_name, mapped_bid, data_torch.ndim):
|
||||
preserve_fp8_e4m3 = False
|
||||
break
|
||||
if preserve_fp8_e4m3:
|
||||
self._fp8_e4m3_preserved.add(name)
|
||||
|
||||
old_dtype = data_torch.dtype
|
||||
|
||||
# convert any unsupported data types to float32
|
||||
if data_torch.dtype not in (torch.float16, torch.float32) and name not in self._fp8_e4m3_preserved:
|
||||
if data_torch.dtype not in (torch.float16, torch.float32):
|
||||
data_torch = data_torch.to(torch.float32)
|
||||
|
||||
# use the first number-like part of the tensor name as the block id
|
||||
@@ -1496,14 +1238,51 @@ class ModelBase:
|
||||
break
|
||||
|
||||
for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)):
|
||||
n_dims = len(data_torch.shape)
|
||||
# TODO: why do we squeeze here?
|
||||
# data = data_torch.squeeze().numpy()
|
||||
data = data_torch.numpy()
|
||||
|
||||
n_dims = len(data.shape)
|
||||
data_qtype: gguf.GGMLQuantizationType | bool = self.tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
# Most of the codebase that takes in 1D tensors or norms only handles F32 tensors
|
||||
if n_dims <= 1 or new_name.endswith("_norm.weight"):
|
||||
data_qtype = gguf.GGMLQuantizationType.F32
|
||||
|
||||
if data_qtype is False and self._tensor_requires_f32(new_name, bid, n_dims):
|
||||
# Conditions should closely match those in llama_model_quantize_internal in llama.cpp
|
||||
# Some tensor types are always in float32
|
||||
if data_qtype is False and (
|
||||
any(
|
||||
self.match_model_tensor_name(new_name, key, bid)
|
||||
for key in (
|
||||
gguf.MODEL_TENSOR.FFN_GATE_INP,
|
||||
gguf.MODEL_TENSOR.FFN_GATE_INP_SHEXP,
|
||||
gguf.MODEL_TENSOR.POS_EMBD,
|
||||
gguf.MODEL_TENSOR.TOKEN_TYPES,
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D,
|
||||
gguf.MODEL_TENSOR.SHORTCONV_CONV,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_FIRST,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_W1,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_W2,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_DECAY_W1,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_DECAY_W2,
|
||||
gguf.MODEL_TENSOR.TIME_MIX_LERP_FUSED,
|
||||
gguf.MODEL_TENSOR.POSNET_NORM1,
|
||||
gguf.MODEL_TENSOR.POSNET_NORM2,
|
||||
gguf.MODEL_TENSOR.V_ENC_EMBD_POS,
|
||||
gguf.MODEL_TENSOR.A_ENC_EMBD_POS,
|
||||
gguf.MODEL_TENSOR.ALTUP_CORRECT_COEF,
|
||||
gguf.MODEL_TENSOR.ALTUP_PREDICT_COEF,
|
||||
# Kimi KDA conv weights should be F32
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D_Q,
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D_K,
|
||||
gguf.MODEL_TENSOR.SSM_CONV1D_V,
|
||||
# DSA indexer weights should be F32
|
||||
gguf.MODEL_TENSOR.INDEXER_PROJ,
|
||||
)
|
||||
)
|
||||
or new_name[-7:] not in (".weight", ".lora_a", ".lora_b")
|
||||
):
|
||||
data_qtype = gguf.GGMLQuantizationType.F32
|
||||
|
||||
if data_qtype is False and any(
|
||||
@@ -1541,22 +1320,16 @@ class ModelBase:
|
||||
else:
|
||||
raise ValueError(f"Unknown file type: {self.ftype.name}")
|
||||
|
||||
if data_torch.dtype == torch.float8_e4m3fn:
|
||||
data = data_torch.view(torch.uint8).numpy() if data_qtype == gguf.GGMLQuantizationType.F8_E4M3 else data_torch.float().numpy()
|
||||
else:
|
||||
data = data_torch.numpy()
|
||||
|
||||
# a chunked tensor quantizes as one chunk at a time, while it is written
|
||||
quantize = data.quantize if isinstance(data, gguf.LazyChunkedTensor) else (
|
||||
lambda qtype, d=data: gguf.quants.quantize(d, qtype))
|
||||
|
||||
if not (data_qtype == gguf.GGMLQuantizationType.F8_E4M3 and data.dtype == np.uint8):
|
||||
try:
|
||||
data = quantize(data_qtype)
|
||||
except gguf.QuantError as e:
|
||||
logger.warning("%s, %s", e, "falling back to F16")
|
||||
data_qtype = gguf.GGMLQuantizationType.F16
|
||||
data = quantize(data_qtype)
|
||||
try:
|
||||
data = quantize(data_qtype)
|
||||
except gguf.QuantError as e:
|
||||
logger.warning("%s, %s", e, "falling back to F16")
|
||||
data_qtype = gguf.GGMLQuantizationType.F16
|
||||
data = quantize(data_qtype)
|
||||
|
||||
shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
|
||||
|
||||
@@ -1597,8 +1370,6 @@ class ModelBase:
|
||||
self.ftype = gguf.LlamaFileType.MOSTLY_NVFP4
|
||||
elif self._is_mxfp4:
|
||||
self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE
|
||||
elif self._fp8_e4m3_preserved:
|
||||
self.ftype = gguf.LlamaFileType.MOSTLY_F8_E4M3
|
||||
|
||||
# Generate parameter weight class (useful for leader boards) if not yet determined
|
||||
if self.metadata.size_label is None and total_params > 0:
|
||||
@@ -2901,13 +2672,7 @@ class TextModel(ModelBase):
|
||||
)
|
||||
|
||||
if vocab.tokenizer_type == MistralTokenizerType.tekken:
|
||||
with open(vocab.fname_tokenizer, encoding="utf-8") as f:
|
||||
pattern = json.load(f)["config"]["pattern"]
|
||||
# newer tekken.json split numbers into right-aligned groups of 3 digits instead of single digits
|
||||
if "\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))" in pattern:
|
||||
self.gguf_writer.add_tokenizer_pre("tekken-num3")
|
||||
else:
|
||||
self.gguf_writer.add_tokenizer_pre("tekken")
|
||||
self.gguf_writer.add_tokenizer_pre("tekken")
|
||||
self.gguf_writer.add_token_merges(
|
||||
vocab.extract_vocab_merges_from_model()
|
||||
)
|
||||
|
||||
@@ -236,12 +236,6 @@ class DeepseekModel(TextModel):
|
||||
class DeepseekV2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK2
|
||||
|
||||
def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
|
||||
# MLA projections do not have runtime weight-scale support.
|
||||
if name.endswith((".q_a_proj.weight", ".q_b_proj.weight", ".kv_a_proj_with_mqa.weight", ".kv_b_proj.weight")):
|
||||
return ()
|
||||
return super()._map_fp8_weight_names(name)
|
||||
|
||||
# TODO @ngxson : remove this when we support MTP for deepseek models
|
||||
skip_mtp = True
|
||||
|
||||
|
||||
+3
-19
@@ -203,24 +203,6 @@ class LFM3VLVisionMixin(MmprojModel):
|
||||
if resize_algo is None:
|
||||
raise ValueError(f"unsupported resample: {resample}")
|
||||
self.gguf_writer.add_vision_image_resize_algo(resize_algo)
|
||||
self._check_video_config()
|
||||
|
||||
# mtmd_get_video_ts_style and the lfm3vl video frame sizing hardcode these, conversion fails on a mismatch
|
||||
_video = {"fps": 2.0, "min_frames": 4, "temporal_patch_size": 2, "size": {"shortest_edge": 4096, "longest_edge": 25165824}}
|
||||
_video_max_frames = 768
|
||||
|
||||
def _check_video_config(self):
|
||||
import json
|
||||
video = self.preprocessor_config.get("video_processor")
|
||||
path = self.dir_model / "video_preprocessor_config.json"
|
||||
if video is None and path.is_file():
|
||||
with open(path, encoding="utf-8") as f:
|
||||
video = json.load(f)
|
||||
if video is None:
|
||||
return
|
||||
for k, v in {**self._video, "max_frames": self._video_max_frames}.items():
|
||||
if video.get(k) != v:
|
||||
raise ValueError(f"unsupported video {k}={video.get(k)!r}, expected {v!r}")
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if "patch_embd" in new_name:
|
||||
@@ -453,6 +435,9 @@ class LFM3AudioMmprojModel(MmprojModel):
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if new_name.startswith("a.conv1d.") or "conv_dw" in new_name or new_name.startswith("a.gen.wav.quant."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# the codec decoder is small and its log-magnitude/phase output does not survive quantization
|
||||
if new_name.startswith("a.gen.wav."):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
@@ -577,7 +562,6 @@ class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel):
|
||||
has_audio_encoder = True
|
||||
|
||||
_audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64}
|
||||
_video_max_frames = 384
|
||||
|
||||
# the audio side is the LFM3-Audio model under the HF-native names
|
||||
_audio_renames = (
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("LimiteForCausalLM")
|
||||
@ModelBase.example("paradigma-inc/limite-1b-violetto")
|
||||
class LimiteModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.LIMITE
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.hparams
|
||||
|
||||
expected = {
|
||||
"pos_mode": "rope",
|
||||
"qk_norm": "rms_pre_rope",
|
||||
"rope_style": "interleaved_pairs_odd_lane_sign_flip",
|
||||
"rope_per_layer": False,
|
||||
"rms_norm_has_weight": False,
|
||||
"rms_norm_eps_mode": "torch_finfo_default",
|
||||
"mlp_type": "swiglu",
|
||||
"attn_gate_applied": "per_head_before_o_proj",
|
||||
"ve_head_slice": "first_num_key_value_heads",
|
||||
"ve_applied_before_qk_norm": True,
|
||||
"global_nope": True,
|
||||
"global_window": -1,
|
||||
"mudd_r_site": "resid",
|
||||
"tie_word_embeddings": True,
|
||||
"final_softcap": 0.0,
|
||||
}
|
||||
for key, value in expected.items():
|
||||
if hp.get(key) != value:
|
||||
raise NotImplementedError(f"Limite: unsupported {key}={hp.get(key)!r}, expected {value!r}")
|
||||
if hp["softcap_logits"]["kind"] != "sigmoid":
|
||||
raise NotImplementedError(f"Limite: unsupported softcap_logits kind {hp['softcap_logits']['kind']!r}")
|
||||
|
||||
self.n_head = hp["num_attention_heads"]
|
||||
self.n_head_kv = hp["num_key_value_heads"]
|
||||
self.head_dim = hp["head_dim"]
|
||||
self.n_pairs = hp["rope_n_pairs"]
|
||||
|
||||
self.mudd_taps: dict[int, list[int]] = {}
|
||||
if hp.get("mudd"):
|
||||
if not hp.get("mudd_mlp"):
|
||||
raise NotImplementedError("Limite: mudd without mudd_mlp is not supported")
|
||||
self.mudd_taps = {int(il): [int(t) for t in taps] for il, taps in hp["mudd_tap_idx"].items()}
|
||||
assert all(len(taps) == hp["mudd_taps"] for taps in self.mudd_taps.values())
|
||||
|
||||
def set_vocab(self):
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
hp = self.hparams
|
||||
|
||||
self.gguf_writer.add_key_length(self.head_dim)
|
||||
self.gguf_writer.add_value_length(self.head_dim)
|
||||
|
||||
# torch rms_norm with eps=None uses the fp32 epsilon, even for bf16 inputs
|
||||
self.gguf_writer.add_layer_norm_rms_eps(torch.finfo(torch.float32).eps)
|
||||
self.gguf_writer.add_attention_scale(hp["attention_softmax_scale"])
|
||||
self.gguf_writer.add_xsa_eps(hp["xsa_normalize_eps"] if hp.get("xsa") else 0.0)
|
||||
|
||||
# HF freq_i = base^(-i/(n_pairs-1)), ggml uses freq_base^(-i/n_pairs)
|
||||
self.gguf_writer.add_rope_dimension_count(2 * self.n_pairs)
|
||||
self.gguf_writer.add_rope_freq_base(hp["rope_base_local"] ** (self.n_pairs / (self.n_pairs - 1)))
|
||||
|
||||
# HF keeps keys k >= q - window, so a window of W spans W + 1 tokens
|
||||
global_layers = set(hp["global_layers"])
|
||||
self.gguf_writer.add_sliding_window(hp["sliding_window"] + 1)
|
||||
self.gguf_writer.add_sliding_window_pattern([il not in global_layers for il in range(self.block_count)])
|
||||
|
||||
cap = hp["softcap_logits"]
|
||||
self.gguf_writer.add_final_logit_sigmoid_capping(cap["a"], cap["b"], cap["c"])
|
||||
|
||||
if self.mudd_taps:
|
||||
n_taps = hp["mudd_taps"]
|
||||
tap_indices = [-1] * (self.block_count * n_taps)
|
||||
for il, taps in self.mudd_taps.items():
|
||||
tap_indices[il * n_taps:(il + 1) * n_taps] = taps
|
||||
self.gguf_writer.add_mudd_feed_forward_length(hp["mudd_inter"])
|
||||
self.gguf_writer.add_mudd_tap_count(n_taps)
|
||||
self.gguf_writer.add_mudd_tap_indices(tap_indices)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# tiny mixing weights, kept at full precision as in the reference
|
||||
# the mudd tensors come from a global source tensor, so take the block id from the new name
|
||||
if new_name.startswith("blk."):
|
||||
bid = int(new_name.split(".")[1])
|
||||
for key in (gguf.MODEL_TENSOR.ATTN_GATE, gguf.MODEL_TENSOR.ATTN_VE_GATE, gguf.MODEL_TENSOR.MUDD_DOWN, gguf.MODEL_TENSOR.MUDD_UP):
|
||||
if self.match_model_tensor_name(new_name, key, bid):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def _swap_rope_pairs(self, w: Tensor, n_head: int) -> Tensor:
|
||||
# HF rotates each pair by -theta; swapping the pair lanes gives the ggml +theta rotation.
|
||||
# q and k are both swapped, so the attention scores do not change.
|
||||
idx = torch.arange(self.head_dim)
|
||||
idx[:2 * self.n_pairs] = idx[:2 * self.n_pairs].view(-1, 2).flip(-1).reshape(-1)
|
||||
w = w.reshape(n_head, self.head_dim, -1)
|
||||
return w[:, idx, :].reshape(n_head * self.head_dim, -1)
|
||||
|
||||
def _pad_cols(self, w: Tensor) -> Tensor:
|
||||
# gates read only the first channels of the input, pad them to n_embd
|
||||
return torch.nn.functional.pad(w, (0, self.hparams["hidden_size"] - w.shape[-1]))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
hp = self.hparams
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def load(n: str) -> Tensor:
|
||||
return self.model_tensors[n]().float()
|
||||
|
||||
if name == "model.value_embeds.weight":
|
||||
ve = data_torch.reshape(data_torch.shape[0], -1, hp["ve_dim"])[:, :self.n_head_kv]
|
||||
ve = torch.nn.functional.pad(ve, (0, self.head_dim - hp["ve_dim"]))
|
||||
yield self.format_tensor_name(T.VALUE_EMBD), ve.reshape(ve.shape[0], -1) * hp["ve_gate_scale"]
|
||||
return
|
||||
|
||||
if name == "model.mudd.dense1":
|
||||
for il in self.mudd_taps:
|
||||
yield self.format_tensor_name(T.MUDD_DOWN, il), data_torch
|
||||
return
|
||||
if name == "model.mudd.dense2":
|
||||
# rows [0, n) mix the attention input, rows [n, 2n) mix the residual
|
||||
dense2_mlp = load("model.mudd.dense2_mlp")
|
||||
bias = load("model.mudd.bias")
|
||||
bias_mlp = load("model.mudd.bias_mlp")
|
||||
for il, taps in self.mudd_taps.items():
|
||||
n = len(taps)
|
||||
w = torch.cat([data_torch[il, :n], dense2_mlp[il, :n]], 0)
|
||||
b = torch.cat([bias[il, :n], bias_mlp[il, :n]], 0)
|
||||
yield self.format_tensor_name(T.MUDD_UP, il), w
|
||||
yield self.format_tensor_name(T.MUDD_UP, il, ".bias"), b
|
||||
return
|
||||
if name.startswith("model.mudd."):
|
||||
return
|
||||
|
||||
if bid is None:
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
return
|
||||
|
||||
prefix = f"model.layers.{bid}."
|
||||
name = name.removeprefix(prefix)
|
||||
|
||||
if name in ("self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"):
|
||||
data_torch = data_torch * load(prefix + "self_attn.qkv_scale")
|
||||
if name == "self_attn.q_proj.weight":
|
||||
data_torch = self._swap_rope_pairs(data_torch, self.n_head)
|
||||
elif name == "self_attn.k_proj.weight":
|
||||
data_torch = self._swap_rope_pairs(data_torch, self.n_head_kv)
|
||||
elif name == "self_attn.o_proj.weight":
|
||||
data_torch = data_torch * load(prefix + "self_attn.o_scale") * load(prefix + "post_lambda_attn") * hp["attn_gate_scale"]
|
||||
elif name == "mlp.down_proj.weight":
|
||||
data_torch = data_torch * load(prefix + "post_lambda_mlp")
|
||||
elif name == "self_attn.attn_gate":
|
||||
yield self.format_tensor_name(T.ATTN_GATE, bid), self._pad_cols(data_torch)
|
||||
return
|
||||
elif name == "self_attn.ve_gate":
|
||||
yield self.format_tensor_name(T.ATTN_VE_GATE, bid), self._pad_cols(data_torch[:self.n_head_kv])
|
||||
return
|
||||
elif name == "self_attn.xsa_alpha":
|
||||
if hp.get("xsa") and bid in hp["xsa_layers"]:
|
||||
yield self.format_tensor_name(T.ATTN_XSA_ALPHA, bid), torch.tanh(data_torch)
|
||||
return
|
||||
elif name == "resid_lambda_attn":
|
||||
yield self.format_tensor_name(T.ATTN_RESID_SCALE, bid), data_torch.reshape(1)
|
||||
return
|
||||
elif name == "resid_lambda_mlp":
|
||||
yield self.format_tensor_name(T.FFN_RESID_SCALE, bid), data_torch.reshape(1)
|
||||
return
|
||||
elif name in ("self_attn.qkv_scale", "self_attn.o_scale", "post_lambda_attn", "post_lambda_mlp"):
|
||||
# folded into the projections above
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, prefix + name, bid)
|
||||
@@ -1,217 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
|
||||
from .mistral import MistralModel, MistralMoeModel
|
||||
|
||||
|
||||
@ModelBase.register("MistralLarge4ForConditionalGeneration")
|
||||
class MistralLarge4Model(MistralMoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.MISTRAL_LARGE4
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
n_layer = self.hparams["num_hidden_layers"]
|
||||
dsa = self.hparams["dsa"]
|
||||
|
||||
def expand(pattern: str) -> list[str]:
|
||||
values = pattern.split(",")
|
||||
return [values[i % len(values)] for i in range(n_layer)]
|
||||
|
||||
self.layer_has_rope = [v == "rope" for v in expand(self.hparams.get("nope") or "rope")]
|
||||
self.layer_indexer_full = [v == "full" for v in expand(dsa.get("sharing") or "full")]
|
||||
|
||||
kv_sharing = self.hparams.get("kv_sharing") or {}
|
||||
if "source_layers_str" in kv_sharing:
|
||||
self.layer_kv_source = [int(v) for v in kv_sharing["source_layers_str"].split(",")]
|
||||
else:
|
||||
assert not kv_sharing, f"unsupported kv_sharing config: {kv_sharing}"
|
||||
self.layer_kv_source = list(range(n_layer))
|
||||
assert len(self.layer_kv_source) == n_layer
|
||||
|
||||
# note: the rope dim of the indexer must match the main attention
|
||||
assert dsa.get("rope_head_dim", -1) in (-1, self.hparams["qk_rope_head_dim"])
|
||||
assert dsa.get("match_nope", False), "indexer must follow the NoPE pattern of the main attention"
|
||||
assert dsa.get("ssmax_scaling", False), "indexer without SSMax is not supported"
|
||||
assert self.hparams.get("q_lora_rank") is None
|
||||
assert not self.hparams["yarn"].get("apply_scale", True), "only YaRN without magnitude scaling is supported"
|
||||
|
||||
def dequant_model(self):
|
||||
quant_config = self.hparams.get("quantization")
|
||||
if quant_config is not None:
|
||||
assert quant_config["qformat_weight"] == "fp8_e4m3"
|
||||
assert quant_config.get("qscheme_weight") == "BLOCK"
|
||||
self.hparams["quantization_config"] = {
|
||||
"quant_method": "fp8",
|
||||
"weight_block_size": [quant_config["block_size_out"], quant_config["block_size_in"]],
|
||||
}
|
||||
return super().dequant_model()
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
# skip the MistralMoeModel overrides, they apply the deepseek2 YaRN scaling
|
||||
super(MistralMoeModel, self).set_gguf_parameters()
|
||||
MistralModel.set_mistral_config(self.gguf_writer, self.hparams)
|
||||
|
||||
dsa = self.hparams["dsa"]
|
||||
self.gguf_writer.add_indexer_head_count(dsa["n_heads"])
|
||||
self.gguf_writer.add_indexer_key_length(dsa["head_dim"])
|
||||
self.gguf_writer.add_indexer_top_k(dsa["topk"])
|
||||
self.gguf_writer.add_indexer_types(self.layer_indexer_full)
|
||||
self.gguf_writer.add_layer_norm_eps(dsa.get("norm_eps", 1e-6))
|
||||
|
||||
self.gguf_writer.add_rope_pattern(self.layer_has_rope)
|
||||
self.gguf_writer.add_kv_source_map(self.layer_kv_source)
|
||||
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
|
||||
self.gguf_writer.add_final_logit_softcapping(self.hparams["softmax_tanh_gating"])
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# exported separately with --dspark
|
||||
if name.startswith("mtp_"):
|
||||
return None
|
||||
|
||||
# experts need HF names for the merging logic in DeepseekV2Model
|
||||
# note: bare "layers.N.experts" comes from the NVFP4 quantized_layers config
|
||||
if ".experts." in name or name.endswith(".experts"):
|
||||
name = "model." + name.replace(".experts", ".mlp.experts")
|
||||
name = name.replace(".w1.", ".gate_proj.").replace(".w2.", ".down_proj.").replace(".w3.", ".up_proj.")
|
||||
|
||||
# skip the MistralMoeModel renaming, the qscale names are needed for block FP8 dequant
|
||||
return super(MistralMoeModel, cls).filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name == "lerope.alpha":
|
||||
# LeRoPE scales inv_freq by exp(alpha), ggml divides theta by the freq factors
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.exp(-data_torch.float()))
|
||||
return
|
||||
|
||||
if bid is not None and name.endswith((".ssmax_scaling.s", ".ssmax_scaling.b")):
|
||||
tensor = gguf.MODEL_TENSOR.INDEXER_SSMAX if ".indexer." in name else gguf.MODEL_TENSOR.ATTN_SSMAX
|
||||
suffix = ".weight" if name.endswith(".s") else ".bias"
|
||||
yield (self.format_tensor_name(tensor, bid, suffix), data_torch.float())
|
||||
return
|
||||
|
||||
if bid is not None and ".indexer.q_input_norm." in name:
|
||||
suffix = ".weight" if name.endswith(".weight") else ".bias"
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.INDEXER_Q_NORM, bid, suffix), data_torch)
|
||||
return
|
||||
|
||||
if bid is not None and ".indexer." in name:
|
||||
name = "model." + name.replace(".attention.", ".self_attn.")
|
||||
|
||||
# NoPE layers have a larger non-rotary head, so wkv_b is split per layer
|
||||
if bid is not None and name.endswith(".wkv_b.weight"):
|
||||
n_head = self.hparams["num_attention_heads"]
|
||||
v_head_dim = self.hparams["v_head_dim"]
|
||||
qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
|
||||
if not self.layer_has_rope[bid]:
|
||||
qk_nope_head_dim += self.hparams["qk_rope_head_dim"]
|
||||
|
||||
assert data_torch.shape[0] == n_head * (qk_nope_head_dim + v_head_dim)
|
||||
|
||||
kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1])
|
||||
k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
|
||||
k_b = k_b.transpose(1, 2)
|
||||
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid), k_b)
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid), v_b)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
class MistralLarge4DSparkModel(MistralLarge4Model):
|
||||
"""Mistral Large 4 DSpark draft (mtp_dspark.*) as a DFLASH GGUF, export with --dspark"""
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
_DSPARK_ROOT: dict[str, gguf.MODEL_TENSOR] = {
|
||||
"backbone.target_projection.weight": gguf.MODEL_TENSOR.FC,
|
||||
"backbone.target_norm.weight": gguf.MODEL_TENSOR.ENC_OUTPUT_NORM,
|
||||
"backbone.final_norm.weight": gguf.MODEL_TENSOR.OUTPUT_NORM,
|
||||
"backbone.mask_embedding": gguf.MODEL_TENSOR.DSPARK_MASK_EMBD,
|
||||
"autoregressive_head.embedding.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W1,
|
||||
"autoregressive_head.output.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W2,
|
||||
"autoregressive_head.joint_proj.weight": gguf.MODEL_TENSOR.DSPARK_JOINT_PROJ,
|
||||
"autoregressive_head.joint_proj.bias": gguf.MODEL_TENSOR.DSPARK_JOINT_PROJ,
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
mtp = self.hparams.get("mtp") or {}
|
||||
dspark = mtp.get("dspark")
|
||||
if dspark is None:
|
||||
raise ValueError("Mistral Large 4 DSpark export needs the mtp.dspark config")
|
||||
assert dspark.get("head_type") == "rnn", f"unsupported DSpark head type: {dspark.get('head_type')}"
|
||||
assert dspark.get("draft_input") == "position_specific", f"unsupported DSpark draft input: {dspark.get('draft_input')}"
|
||||
|
||||
windows = {int(v) for v in (mtp.get("ragged_attention") or "").split(",") if v.strip()}
|
||||
assert len(windows) == 1, f"DSpark draft layers must share one sliding window, got {mtp.get('ragged_attention')}"
|
||||
|
||||
self._n_stage = int(mtp["n_layers"])
|
||||
self._depth = int(mtp["depth"])
|
||||
self._window = windows.pop()
|
||||
self._target_layers = [int(v) for v in mtp["target_activation_layers"].split(",")]
|
||||
|
||||
self.layer_has_rope = [True] * self._n_stage
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if not name.startswith("mtp_dspark."):
|
||||
return None
|
||||
name = name.removeprefix("mtp_dspark.")
|
||||
if name.startswith("backbone.layers."):
|
||||
return name.removeprefix("backbone."), gen
|
||||
return f"dspark.{name}", gen
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
# block uses mask_embd, mask token id is not used
|
||||
self.gguf_writer.add_mask_token_id(0)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
w = self.gguf_writer
|
||||
w.add_block_count(self._n_stage)
|
||||
w.add_context_length(hp["max_position_embeddings"])
|
||||
w.add_embedding_length(hp["hidden_size"])
|
||||
w.add_feed_forward_length(hp["intermediate_size"])
|
||||
w.add_head_count(hp["num_attention_heads"])
|
||||
w.add_head_count_kv(1)
|
||||
w.add_key_length(hp["kv_lora_rank"] + hp["qk_rope_head_dim"])
|
||||
w.add_value_length(hp["kv_lora_rank"])
|
||||
w.add_key_length_mla(hp["qk_nope_head_dim"] + hp["qk_rope_head_dim"])
|
||||
w.add_value_length_mla(hp["v_head_dim"])
|
||||
w.add_kv_lora_rank(hp["kv_lora_rank"])
|
||||
w.add_rope_dimension_count(hp["qk_rope_head_dim"])
|
||||
w.add_rope_freq_base(float(hp["rope_theta"]))
|
||||
w.add_layer_norm_rms_eps(hp["rms_norm_eps"])
|
||||
MistralModel.set_mistral_config(w, hp)
|
||||
w.add_sliding_window(self._window)
|
||||
w.add_final_logit_softcapping(hp["softmax_tanh_gating"])
|
||||
w.add_block_size(self._depth)
|
||||
# draft reads layer output = input of next layer
|
||||
w.add_target_layers([layer + 1 for layer in self._target_layers])
|
||||
w.add_sample_from_anchor(True)
|
||||
w.add_has_confidence_head(False)
|
||||
w.add_file_type(self.ftype)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("dspark."):
|
||||
key = name.removeprefix("dspark.")
|
||||
if key not in self._DSPARK_ROOT:
|
||||
raise ValueError(f"Unexpected DSpark tensor {name!r}")
|
||||
suffix = ".bias" if key.endswith(".bias") else ".weight"
|
||||
yield (self.format_tensor_name(self._DSPARK_ROOT[key], None, suffix), data_torch)
|
||||
return
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
@@ -433,11 +433,6 @@ class NemotronHModel(GraniteHybridModel):
|
||||
_ATTN_LAYER_TYPES = {"attention", "full_attention"}
|
||||
_MLP_LAYER_TYPES = {"moe"}
|
||||
|
||||
def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
|
||||
if ".mixer.experts." in name:
|
||||
name = name.replace("backbone.", "model.").replace(".mixer.experts.", ".mlp.experts.")
|
||||
return super()._map_fp8_weight_names(name)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
# We have to determine the correct model architecture (MoE vs non-MoE) before
|
||||
# calling the parent __init__. This is because the parent constructor
|
||||
|
||||
@@ -83,15 +83,6 @@ class Qwen2Model(TextModel):
|
||||
class Qwen2MoeModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2MOE
|
||||
|
||||
def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
|
||||
if name.removesuffix(".weight").endswith(".mlp.experts.gate_up_proj"):
|
||||
bid = next(int(part) for part in self.map_tensor_name(name).split(".") if part.isdecimal())
|
||||
return (
|
||||
self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_EXP, bid),
|
||||
self.format_tensor_name(gguf.MODEL_TENSOR.FFN_UP_EXP, bid),
|
||||
)
|
||||
return super()._map_fp8_weight_names(name)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
|
||||
@@ -391,16 +382,6 @@ class _QwenMtpMixin:
|
||||
class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3NEXT
|
||||
|
||||
def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
|
||||
if name.endswith(".linear_attn.in_proj_qkvz.weight"):
|
||||
bid = next(int(part) for part in self.map_tensor_name(name).split(".") if part.isdecimal())
|
||||
# Both split projections keep the original per-tensor scales.
|
||||
return (
|
||||
self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid),
|
||||
self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_GATE, bid),
|
||||
)
|
||||
return super()._map_fp8_weight_names(name)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_ssm_conv_kernel(self.hparams["linear_conv_kernel_dim"])
|
||||
|
||||
+5
-14
@@ -58,7 +58,7 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--outtype", type=str, choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"], default="auto",
|
||||
help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto to preserve supported FP8 weights and otherwise select the highest-fidelity 16-bit float type",
|
||||
help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bigendian", action="store_true",
|
||||
@@ -127,7 +127,7 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dspark", action="store_true",
|
||||
help="Export only the DeepSeek-V4 or Mistral Large 4 DSpark draft tensors as a separate GGUF.",
|
||||
help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mistral-format", action="store_true",
|
||||
@@ -255,9 +255,6 @@ def main() -> None:
|
||||
assert hparams.get("vision_encoder") is not None, "This model does not support multimodal"
|
||||
from conversion.pixtral import PixtralModel
|
||||
model_class = PixtralModel
|
||||
elif hparams.get("moe") is not None and hparams.get("dsa") is not None:
|
||||
from conversion.mistral4 import MistralLarge4Model
|
||||
model_class = MistralLarge4Model
|
||||
elif hparams.get("moe") is not None:
|
||||
from conversion.mistral import MistralMoeModel
|
||||
model_class = MistralMoeModel
|
||||
@@ -270,16 +267,10 @@ def main() -> None:
|
||||
sys.exit(1)
|
||||
|
||||
if args.dspark:
|
||||
if is_mistral_format or model_architecture == "MistralLarge4ForConditionalGeneration":
|
||||
if model_class.__name__ != "MistralLarge4Model":
|
||||
logger.error("--dspark is only supported for Mistral Large 4 among Mistral-format models")
|
||||
sys.exit(1)
|
||||
from conversion.mistral4 import MistralLarge4DSparkModel
|
||||
model_class = MistralLarge4DSparkModel
|
||||
elif model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"):
|
||||
logger.error("--dspark is only supported for DeepseekV4ForCausalLM, DeepseekV41ForCausalLM and Mistral Large 4")
|
||||
if is_mistral_format or model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"):
|
||||
logger.error("--dspark is only supported for DeepseekV4ForCausalLM and DeepseekV41ForCausalLM")
|
||||
sys.exit(1)
|
||||
elif model_architecture == "DeepseekV41ForCausalLM":
|
||||
if model_architecture == "DeepseekV41ForCausalLM":
|
||||
from conversion.deepseek41 import DeepseekV41DSparkModel
|
||||
model_class = DeepseekV41DSparkModel
|
||||
else:
|
||||
|
||||
@@ -784,7 +784,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
|
||||
| GGML_SYCL_TARGET | INTEL *(default)* | Set the SYCL target device type. |
|
||||
| GGML_SYCL_DEVICE_ARCH | Optional | Set the SYCL device architecture. Setting the device architecture can improve the performance. See the table [--offload-arch](https://github.com/intel/llvm/blob/sycl/sycl/doc/design/OffloadDesign.md#--offload-arch) for a list of valid architectures. |
|
||||
| GGML_SYCL_F16 | OFF *(default)* \|ON *(optional)* | Enable FP16 build with SYCL code path. [1.](#sycl-f16-footnote) |
|
||||
| GGML_SYCL_SUPPORT_XMX | ON *(default)* \|OFF *(Optional)* | Build the code paths that use XMX (DPAS) instructions. Set OFF for AOT builds targeting iGPUs without XMX, which otherwise fail to compile. |
|
||||
| GGML_SYCL_GRAPH | ON *(default)* \|OFF *(Optional)* | Enable build with [SYCL Graph extension](https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc). |
|
||||
| GGML_SYCL_DNN | ON *(default)* \|OFF *(Optional)* | Enable build with oneDNN. |
|
||||
| GGML_SYCL_HOST_MEM_FALLBACK | ON *(default)* \|OFF *(Optional)* | Allow host memory fallback when device memory is full during quantized weight reorder. Enables inference to continue at reduced speed (reading over PCIe) instead of failing. Requires Linux kernel 6.8+. |
|
||||
@@ -813,7 +812,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
|
||||
| GGML_SYCL_ENABLE_DNN | 0 or 1 (default)| Enable running computations through oneDNN and always use oneMKL. |
|
||||
| GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. |
|
||||
| GGML_SYCL_FA_ONEDNN_MAX_KV | 0 (default, disabled) or positive integer | By default (0), all sequences are handled by the oneDNN fused SDPA path, regardless of KV length; a positive value caps that length, past which sequences fall back to the native kernel. If GPU driver watchdog resets (DEVICE_LOST) occur during long-context inference, set this near the context depth where they start, e.g. 24576. |
|
||||
| GGML_SYCL_FA_DPAS | 1 (default) or 0 | Enable the XMX flash-attention kernel for a q4_0 or q8_0 KV cache (`-ctk q4_0 -ctv q4_0` or `-ctk q8_0 -ctv q8_0`) on Xe2 or newer GPUs (Arc B-series, Lunar Lake, Panther Lake, Wildcat Lake). It reads the quantized cache directly for decode and batches of up to 31 tokens, head sizes 64, 96, 128, 256 and 512, up to 32 query heads per KV head. Set to 0 to use the native SYCL flash-attention kernels. |
|
||||
| GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. |
|
||||
| GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` |
|
||||
| GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. |
|
||||
@@ -848,18 +846,6 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
|
||||
Then benchmark with your preferred tool, such as `llama-bench`.
|
||||
For graph activity details (including reuse), set `GGML_SYCL_DEBUG=1` and look for `[SYCL-GRAPH]` messages.
|
||||
|
||||
### Flash-attention kernels
|
||||
|
||||
Attention dispatches on shape and hardware.
|
||||
|
||||
DPAS is the quantized path. q4_0 or q8_0 KV cache, decode and batches up to 31 tokens, Xe2 and newer only. It issues dpas straight on the quantized blocks and never materialises an f16 copy. vISA gets used here because joint_matrix has no s4 type.
|
||||
|
||||
oneDNN does XMX through SDPA but converts KV to f16 first, so it ends up on prefill: 32+ tokens and 1K+ keys. MKL is the prompt processing equivalent through oneMKL GEMM, also f16.
|
||||
|
||||
VEC/TILE are plain kernels with no XMX in them. They run when DPAS is off or the hardware doesn't support it.
|
||||
|
||||
joint_matrix belongs to the matmul GEMMs. Attention doesn't use it.
|
||||
|
||||
## Compile-time Flags
|
||||
|
||||
Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spot.
|
||||
|
||||
+47
-48
@@ -14,22 +14,22 @@ Legend:
|
||||
|
||||
| Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
|
||||
|-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|
|
||||
| ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
@@ -38,49 +38,49 @@ Legend:
|
||||
| CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG_MASK_INF | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_PRE | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
|
||||
| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | ❌ |
|
||||
| MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
|
||||
| MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | 🟡 |
|
||||
| MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 |
|
||||
| MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
|
||||
| MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
@@ -89,46 +89,45 @@ Legend:
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
|
||||
| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
|
||||
| STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ |
|
||||
| STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| TOP_K_MASK | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
|
||||
+16661
-20425
File diff suppressed because it is too large
Load Diff
+2
-3
@@ -100,9 +100,8 @@ llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DSpark.gguf \
|
||||
|
||||
`--spec-draft-n-max` is clamped to the draft model's trained block size.
|
||||
|
||||
`--spec-draft-p-min P` truncates each drafted block at the first position whose predicted
|
||||
acceptance (from the draft's confidence head) falls below `P` (default 0 = disabled).
|
||||
Values above 0 require a draft with a confidence head.
|
||||
`--spec-draft-conf-min P` truncates each drafted block at the first position whose predicted
|
||||
acceptance (from the draft's confidence head, if present) falls below `P` (default 0 = disabled).
|
||||
|
||||
Currently only drafts with a Qwen3 backbone are supported; support for other backbones
|
||||
(e.g. Gemma4) is planned.
|
||||
|
||||
@@ -15,10 +15,6 @@ Without --instructions, a forced mode comes with the system prompt of the LFM3-A
|
||||
|
||||
python examples/realtime_client.py --output-modalities text,audio --out answer.wav question.wav
|
||||
|
||||
Tool calls: --tools gives the conversation a demo tool, the client runs its calls and sends the results back:
|
||||
|
||||
python examples/realtime_client.py --tools --text "Set the volume to 30."
|
||||
|
||||
Text turns go in with --text, transcription sessions transcribe each file:
|
||||
|
||||
python examples/realtime_client.py --text "What is the capital of France?"
|
||||
@@ -35,7 +31,6 @@ Requirements: pip install "openai[realtime]" numpy
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import wave
|
||||
|
||||
@@ -48,26 +43,6 @@ MODE_INSTRUCTIONS = {
|
||||
"audio": {"text,audio": "Respond with interleaved text and audio."},
|
||||
"omni": {"text": "Respond in text.", "text,audio": "Respond in interleaved text and audio."},
|
||||
}
|
||||
# and with tools: calls, or an answer in interleaved text and speech
|
||||
TOOLS_INSTRUCTIONS = "Respond with function calls, and interleaved text and audio.\n\nList of tools:"
|
||||
|
||||
# the demo tool of --tools
|
||||
DEMO_TOOLS = [{
|
||||
"type": "function",
|
||||
"name": "set_volume",
|
||||
"description": "Set the device speaker volume.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"volume": {"type": "integer", "description": "Volume from 0 to 100."}},
|
||||
"required": ["volume"],
|
||||
},
|
||||
}]
|
||||
|
||||
|
||||
def run_tool(name, arguments):
|
||||
if name == "set_volume":
|
||||
return {"volume": json.loads(arguments)["volume"]}
|
||||
return {"error": f"unknown tool {name}"}
|
||||
|
||||
|
||||
def read_wav(path, rate):
|
||||
@@ -111,9 +86,7 @@ async def main(args):
|
||||
if args.type == "realtime":
|
||||
instructions = args.instructions
|
||||
if instructions is None:
|
||||
instructions = TOOLS_INSTRUCTIONS if args.tools else MODE_INSTRUCTIONS[card].get(args.output_modalities)
|
||||
if args.tools:
|
||||
session["tools"] = DEMO_TOOLS
|
||||
instructions = MODE_INSTRUCTIONS[card].get(args.output_modalities)
|
||||
if instructions is not None:
|
||||
session["instructions"] = instructions
|
||||
if args.output_modalities:
|
||||
@@ -163,18 +136,7 @@ async def main(args):
|
||||
elif ev.type == "response.output_audio.delta":
|
||||
t_first = t_first or time.perf_counter()
|
||||
audio.append(np.frombuffer(base64.b64decode(ev.delta), dtype=np.int16))
|
||||
elif ev.type == "response.done":
|
||||
# the calls of the response are run, their results go back and the model answers them
|
||||
calls = [o for o in ev.model_dump()["response"]["output"] if o["type"] == "function_call"]
|
||||
for call in calls:
|
||||
result = json.dumps(run_tool(call["name"], call["arguments"]))
|
||||
print(f"\n[call {call['name']}({call['arguments']}) -> {result}]", flush=True)
|
||||
await conn.conversation.item.create(item={
|
||||
"type": "function_call_output", "call_id": call["call_id"], "output": result})
|
||||
if not calls:
|
||||
break
|
||||
await conn.response.create()
|
||||
elif ev.type == "conversation.item.input_audio_transcription.completed":
|
||||
elif ev.type in ("response.done", "conversation.item.input_audio_transcription.completed"):
|
||||
break
|
||||
elif ev.type == "error":
|
||||
print("\nerror:", ev.error.message)
|
||||
@@ -199,7 +161,6 @@ if __name__ == "__main__":
|
||||
p.add_argument("--text", action="append", default=[], help="a user turn as text, after the audio turns, repeatable")
|
||||
p.add_argument("--type", choices=["realtime", "transcription"], default="realtime", help="session type")
|
||||
p.add_argument("--instructions", help="system prompt of the conversation")
|
||||
p.add_argument("--tools", action="store_true", help="give the conversation a demo tool (set_volume), its calls are run")
|
||||
p.add_argument("--output-modalities", choices=["text", "audio", "text,audio"], help="default: the model picks")
|
||||
p.add_argument("--language", help="transcription language, e.g. en")
|
||||
p.add_argument("--chunk-ms", type=int, default=100, help="audio chunk size")
|
||||
|
||||
+1
-2
@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)
|
||||
|
||||
### GGML Version
|
||||
set(GGML_VERSION_MAJOR 0)
|
||||
set(GGML_VERSION_MINOR 28)
|
||||
set(GGML_VERSION_MINOR 27)
|
||||
set(GGML_VERSION_PATCH 0)
|
||||
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
|
||||
|
||||
@@ -250,7 +250,6 @@ option(GGML_OPENMP_FETCH "ggml: fetch LLVM OpenMP"
|
||||
option(GGML_RPC "ggml: use RPC" OFF)
|
||||
option(GGML_SYCL "ggml: use SYCL" OFF)
|
||||
option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF)
|
||||
option(GGML_SYCL_SUPPORT_XMX "ggml: build the XMX (DPAS) code paths in the SYCL backend" ON)
|
||||
option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON)
|
||||
option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON)
|
||||
option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON)
|
||||
|
||||
@@ -70,8 +70,7 @@ extern "C" {
|
||||
GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
|
||||
|
||||
// tensor copy between different backends
|
||||
GGML_API void ggml_backend_tensor_copy (const struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||
GGML_API void ggml_backend_tensor_copy_n(const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors);
|
||||
GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||
|
||||
//
|
||||
// Backend (stream)
|
||||
@@ -96,8 +95,6 @@ extern "C" {
|
||||
GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||
GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
|
||||
GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
|
||||
GGML_API void ggml_backend_tensor_set_n ( struct ggml_tensor * const * tensors, const void * data, const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors);
|
||||
GGML_API void ggml_backend_tensor_get_n (const struct ggml_tensor * const * tensors, void * data, const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors);
|
||||
GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
|
||||
|
||||
GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
|
||||
|
||||
+1
-5
@@ -381,8 +381,6 @@ extern "C" {
|
||||
GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
|
||||
GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
|
||||
|
||||
typedef struct { uint8_t bits; } ggml_fp8_e4m3_t;
|
||||
|
||||
struct ggml_object;
|
||||
struct ggml_context;
|
||||
struct ggml_cgraph;
|
||||
@@ -432,8 +430,7 @@ extern "C" {
|
||||
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
|
||||
GGML_TYPE_Q1_0 = 41,
|
||||
GGML_TYPE_Q2_0 = 42,
|
||||
GGML_TYPE_F8_E4M3 = 43,
|
||||
GGML_TYPE_COUNT = 44,
|
||||
GGML_TYPE_COUNT = 43,
|
||||
};
|
||||
|
||||
// [TAG_GGML_PREC]
|
||||
@@ -489,7 +486,6 @@ extern "C" {
|
||||
GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors
|
||||
GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors
|
||||
GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors
|
||||
GGML_FTYPE_MOSTLY_F8_E4M3 = 29, // except 1d tensors
|
||||
};
|
||||
|
||||
// available tensor operations:
|
||||
|
||||
@@ -47,9 +47,6 @@ extern "C" {
|
||||
// Backend buffer
|
||||
//
|
||||
|
||||
// TODO: remove `set_tensor`, `get_tensor`, `set_tensor_2d`, `get_tensor_2d` and `cpy_tensor` from the buffer interface
|
||||
// they are superseded by the `set_tensor_n`, `get_tensor_n`, `cpy_tensor_n` interface
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/30881#discussion_r4400299490
|
||||
struct ggml_backend_buffer_i {
|
||||
// (optional) free the buffer
|
||||
void (*free_buffer) (ggml_backend_buffer_t buffer);
|
||||
@@ -71,11 +68,6 @@ extern "C" {
|
||||
void (*clear) (ggml_backend_buffer_t buffer, uint8_t value);
|
||||
// (optional) reset any internal state due to tensor initialization, such as tensor extras
|
||||
void (*reset) (ggml_backend_buffer_t buffer);
|
||||
// (optional) batched tensor data copies
|
||||
void (*set_tensor_n) (ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
|
||||
void (*get_tensor_n) (ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
|
||||
// (optional) batched tensor copies: returns true if all copies were performed, false if unsupported and no copies were performed
|
||||
bool (*cpy_tensor_n) (ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors);
|
||||
};
|
||||
|
||||
struct ggml_backend_buffer {
|
||||
|
||||
@@ -1699,9 +1699,6 @@ static const ggml_backend_buffer_i ggml_backend_meta_buffer_iface = {
|
||||
/* .cpy_tensor = */ nullptr,
|
||||
/* .clear = */ ggml_backend_meta_buffer_clear,
|
||||
/* .reset = */ ggml_backend_meta_buffer_reset,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {
|
||||
|
||||
@@ -402,62 +402,6 @@ size_t ggml_backend_get_max_size(ggml_backend_t backend) {
|
||||
return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));
|
||||
}
|
||||
|
||||
template <typename cb_get_buffer>
|
||||
static void ggml_backend_tensor_buft_groups(size_t n_tensors, cb_get_buffer && get_buffer, std::vector<std::vector<size_t>> & groups) {
|
||||
std::unordered_map<ggml_backend_buffer_type_t, size_t> buft_idx;
|
||||
groups.clear();
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
ggml_backend_buffer_t buf = get_buffer(i);
|
||||
auto it = buft_idx.find(buf->buft);
|
||||
if (it == buft_idx.end()) {
|
||||
buft_idx[buf->buft] = groups.size();
|
||||
groups.emplace_back();
|
||||
}
|
||||
groups[buft_idx[buf->buft]].push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
static bool ggml_backend_tensor_batch_is_2d(
|
||||
const struct ggml_tensor * const * tensors,
|
||||
const size_t * offsets,
|
||||
const size_t * sizes,
|
||||
const size_t * data_offsets,
|
||||
size_t n_tensors,
|
||||
size_t * base_offset,
|
||||
size_t * size,
|
||||
size_t * stride_tensor,
|
||||
size_t * stride_data) {
|
||||
if (n_tensors <= 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const struct ggml_tensor * tensor = tensors[0];
|
||||
const size_t size0 = sizes[0];
|
||||
|
||||
if (size0 == 0 || offsets[1] < offsets[0] || data_offsets[1] < data_offsets[0]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const size_t stride_tensor0 = offsets[1] - offsets[0];
|
||||
const size_t stride_data0 = data_offsets[1] - data_offsets[0];
|
||||
|
||||
for (size_t i = 1; i < n_tensors; i++) {
|
||||
if (tensors[i] != tensor || sizes[i] != size0) {
|
||||
return false;
|
||||
}
|
||||
if (offsets[i] != offsets[0] + i*stride_tensor0 || data_offsets[i] != data_offsets[0] + i*stride_data0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
*base_offset = offsets[0];
|
||||
*size = size0;
|
||||
*stride_tensor = stride_tensor0;
|
||||
*stride_data = stride_data0;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
||||
GGML_ASSERT(backend);
|
||||
GGML_ASSERT(tensor);
|
||||
@@ -602,154 +546,6 @@ void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data,
|
||||
buf->iface.get_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
|
||||
}
|
||||
|
||||
void ggml_backend_tensor_set_n(struct ggml_tensor * const * tensors, const void * data,
|
||||
const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors) {
|
||||
if (n_tensors == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<const void *> data_ptrs(n_tensors);
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
data_ptrs[i] = (const char *) data + data_offsets[i];
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t buf = NULL;
|
||||
bool same_buft = true;
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
GGML_ASSERT(tensors[i] != NULL);
|
||||
ggml_backend_buffer_t tensor_buf = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
|
||||
GGML_ASSERT(tensor_buf != NULL && "tensor buffer not set");
|
||||
GGML_ASSERT(tensors[i]->data != NULL && "tensor not allocated");
|
||||
GGML_ASSERT(offsets[i] + sizes[i] <= ggml_nbytes(tensors[i]) && "tensor write out of bounds");
|
||||
|
||||
if (buf == NULL) {
|
||||
buf = tensor_buf;
|
||||
} else if (buf->buft != tensor_buf->buft) {
|
||||
same_buft = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (same_buft && buf->iface.set_tensor_n != NULL) {
|
||||
buf->iface.set_tensor_n(buf, tensors, data_ptrs.data(), offsets, sizes, n_tensors);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!same_buft) {
|
||||
std::vector<std::vector<size_t>> groups;
|
||||
ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) {
|
||||
return tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
|
||||
}, groups);
|
||||
|
||||
for (size_t g = 0; g < groups.size(); g++) {
|
||||
std::vector<ggml_tensor *> g_tensors;
|
||||
std::vector<size_t> g_offsets;
|
||||
std::vector<size_t> g_sizes;
|
||||
std::vector<size_t> g_data_offsets;
|
||||
g_tensors.reserve(groups[g].size());
|
||||
g_offsets.reserve(groups[g].size());
|
||||
g_sizes.reserve(groups[g].size());
|
||||
g_data_offsets.reserve(groups[g].size());
|
||||
|
||||
for (size_t i : groups[g]) {
|
||||
g_tensors.push_back(tensors[i]);
|
||||
g_offsets.push_back(offsets[i]);
|
||||
g_sizes.push_back(sizes[i]);
|
||||
g_data_offsets.push_back(data_offsets[i]);
|
||||
}
|
||||
|
||||
ggml_backend_tensor_set_n(g_tensors.data(), data, g_offsets.data(), g_sizes.data(), g_data_offsets.data(), g_tensors.size());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
size_t base_offset = 0, size = 0, stride_tensor = 0, stride_data = 0;
|
||||
if (ggml_backend_tensor_batch_is_2d(tensors, offsets, sizes, data_offsets, n_tensors,
|
||||
&base_offset, &size, &stride_tensor, &stride_data)) {
|
||||
ggml_backend_tensor_set_2d(tensors[0], data_ptrs[0], base_offset, size,
|
||||
n_tensors, stride_tensor, stride_data);
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
ggml_backend_tensor_set(tensors[i], data_ptrs[i], offsets[i], sizes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_backend_tensor_get_n(const struct ggml_tensor * const * tensors, void * data,
|
||||
const size_t * offsets, const size_t * sizes, const size_t * data_offsets, size_t n_tensors) {
|
||||
if (n_tensors == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<void *> data_ptrs(n_tensors);
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
data_ptrs[i] = (char *) data + data_offsets[i];
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t buf = NULL;
|
||||
bool same_buft = true;
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
GGML_ASSERT(tensors[i] != NULL);
|
||||
ggml_backend_buffer_t tensor_buf = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
|
||||
GGML_ASSERT(tensor_buf != NULL && "tensor buffer not set");
|
||||
GGML_ASSERT(tensors[i]->data != NULL && "tensor not allocated");
|
||||
GGML_ASSERT(offsets[i] + sizes[i] <= ggml_nbytes(tensors[i]) && "tensor read out of bounds");
|
||||
|
||||
if (buf == NULL) {
|
||||
buf = tensor_buf;
|
||||
} else if (buf->buft != tensor_buf->buft) {
|
||||
same_buft = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (same_buft && buf->iface.get_tensor_n != NULL) {
|
||||
buf->iface.get_tensor_n(buf, tensors, data_ptrs.data(), offsets, sizes, n_tensors);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!same_buft) {
|
||||
std::vector<std::vector<size_t>> groups;
|
||||
ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) {
|
||||
return tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
|
||||
}, groups);
|
||||
|
||||
for (size_t g = 0; g < groups.size(); g++) {
|
||||
std::vector<const ggml_tensor *> g_tensors;
|
||||
std::vector<size_t> g_offsets;
|
||||
std::vector<size_t> g_sizes;
|
||||
std::vector<size_t> g_data_offsets;
|
||||
g_tensors.reserve(groups[g].size());
|
||||
g_offsets.reserve(groups[g].size());
|
||||
g_sizes.reserve(groups[g].size());
|
||||
g_data_offsets.reserve(groups[g].size());
|
||||
|
||||
for (size_t i : groups[g]) {
|
||||
g_tensors.push_back(tensors[i]);
|
||||
g_offsets.push_back(offsets[i]);
|
||||
g_sizes.push_back(sizes[i]);
|
||||
g_data_offsets.push_back(data_offsets[i]);
|
||||
}
|
||||
|
||||
ggml_backend_tensor_get_n(g_tensors.data(), data, g_offsets.data(), g_sizes.data(), g_data_offsets.data(), g_tensors.size());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
size_t base_offset = 0, size = 0, stride_tensor = 0, stride_data = 0;
|
||||
if (ggml_backend_tensor_batch_is_2d(tensors, offsets, sizes, data_offsets, n_tensors,
|
||||
&base_offset, &size, &stride_tensor, &stride_data)) {
|
||||
ggml_backend_tensor_get_2d(tensors[0], data_ptrs[0], base_offset, size,
|
||||
n_tensors, stride_tensor, stride_data);
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
ggml_backend_tensor_get(tensors[i], data_ptrs[i], offsets[i], sizes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
|
||||
GGML_ASSERT(tensor);
|
||||
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
|
||||
@@ -852,59 +648,6 @@ void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_backend_tensor_copy_n(const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) {
|
||||
if (n_tensors == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
GGML_ASSERT(ggml_are_same_layout(srcs[i], dsts[i]) && "cannot copy tensors with different layouts");
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t buf = dsts[0]->view_src ? dsts[0]->view_src->buffer : dsts[0]->buffer;
|
||||
bool same_buft = true;
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
|
||||
if (dst_buf->buft != buf->buft) {
|
||||
same_buft = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// if cpy_tensor_n returns false, it must not have performed any copies
|
||||
if (same_buft && buf->iface.cpy_tensor_n != NULL) {
|
||||
if (buf->iface.cpy_tensor_n(buf, srcs, dsts, n_tensors)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!same_buft) {
|
||||
std::vector<std::vector<size_t>> groups;
|
||||
ggml_backend_tensor_buft_groups(n_tensors, [&](size_t i) {
|
||||
return dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
|
||||
}, groups);
|
||||
|
||||
for (size_t g = 0; g < groups.size(); g++) {
|
||||
std::vector<const ggml_tensor *> g_srcs;
|
||||
std::vector<ggml_tensor *> g_dsts;
|
||||
g_srcs.reserve(groups[g].size());
|
||||
g_dsts.reserve(groups[g].size());
|
||||
|
||||
for (size_t i : groups[g]) {
|
||||
g_srcs.push_back(srcs[i]);
|
||||
g_dsts.push_back(dsts[i]);
|
||||
}
|
||||
|
||||
ggml_backend_tensor_copy_n(g_srcs.data(), g_dsts.data(), g_srcs.size());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
ggml_backend_tensor_copy(srcs[i], dsts[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
|
||||
GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
|
||||
|
||||
@@ -1110,9 +853,6 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
|
||||
/* .cpy_tensor = */ NULL,
|
||||
/* .clear = */ ggml_backend_multi_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
|
||||
@@ -2792,9 +2532,6 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
|
||||
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_cpu_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
|
||||
@@ -2809,9 +2546,6 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
|
||||
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_cpu_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
// CPU backend buffer type
|
||||
|
||||
@@ -1478,9 +1478,6 @@ static const ggml_backend_buffer_i ggml_backend_cann_buffer_interface = {
|
||||
/* .cpy_tensor = */ ggml_backend_cann_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_cann_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -116,9 +116,6 @@ static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {
|
||||
/* .cpy_tensor = */ nullptr,
|
||||
/* .clear = */ ggml_backend_amx_buffer_clear,
|
||||
/* .reset = */ nullptr,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
static const char * ggml_backend_amx_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
|
||||
|
||||
@@ -535,60 +535,6 @@ void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v
|
||||
UNUSED(ncols_interleaved);
|
||||
UNUSED(blocklen);
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
|
||||
if (svcntb() * 8 == 256) {
|
||||
const svbool_t pg16 = svptrue_pat_b8(SV_VL16);
|
||||
const svbool_t pg4 = svptrue_pat_b32(SV_VL4);
|
||||
const svbool_t pgb = svptrue_b8();
|
||||
const svbool_t pgw = svptrue_b32();
|
||||
const svbool_t pg_hi4 = svnot_b_z(pgw, pg4);
|
||||
const svuint8_t m4b = svdup_n_u8(0x0f);
|
||||
const svuint8_t values = svld1_u8(pg16, (const uint8_t *) kvalues_mxfp4);
|
||||
|
||||
// Reorder Q8 four-byte groups as [0,2,4,6,1,3,5,7]
|
||||
const svuint32_t q8_group_index = svsub_n_u32_m(pg_hi4, svindex_u32(0, 2), 7);
|
||||
|
||||
const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
|
||||
float * res_ptr = s;
|
||||
|
||||
for (int x = 0; x < nc / ncols_interleaved; x++) {
|
||||
const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);
|
||||
svfloat32_t sumf = svdup_n_f32(0.0f);
|
||||
|
||||
for (int l = 0; l < nb; l++) {
|
||||
const svint8_t q8 = svld1_s8(pgb, a_ptr[l].qs);
|
||||
const svint8_t q8_even_odd = svreinterpret_s8_u32(svtbl_u32(svreinterpret_u32_s8(q8), q8_group_index));
|
||||
const svuint8_t qmx01 = svld1_u8(pgb, b_ptr[l].qs + 0);
|
||||
const svuint8_t qmx_lo01 = svtbl_u8(values, svand_u8_x(pgb, qmx01, m4b));
|
||||
const svuint8_t qmx_hi01 = svtbl_u8(values, svlsr_n_u8_x(pgb, qmx01, 4));
|
||||
const svuint8_t qmx23 = svld1_u8(pgb, b_ptr[l].qs + 32);
|
||||
const svuint8_t qmx_lo23 = svtbl_u8(values, svand_u8_x(pgb, qmx23, m4b));
|
||||
const svuint8_t qmx_hi23 = svtbl_u8(values, svlsr_n_u8_x(pgb, qmx23, 4));
|
||||
|
||||
svint32_t sumi = svdup_n_s32(0);
|
||||
sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_lo01), q8_even_odd, 0);
|
||||
sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_lo23), q8_even_odd, 1);
|
||||
sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_hi01), q8_even_odd, 2);
|
||||
sumi = svdot_lane_s32(sumi, svreinterpret_s8_u8(qmx_hi23), q8_even_odd, 3);
|
||||
const svint32_t sumi_cols = svadd_s32_z(pg4, sumi, svext_s32(sumi, sumi, 4));
|
||||
|
||||
// Vector form of GGML_CPU_E8M0_TO_FP32_HALF
|
||||
const svuint32_t e_u32 = svld1ub_u32(pg4, b_ptr[l].e);
|
||||
const svbool_t e_is_denormal = svcmplt_n_u32(pg4, e_u32, 2);
|
||||
const svuint32_t e_denormal_bits = svlsl_u32_x(pg4, svdup_n_u32(0x00200000), e_u32);
|
||||
const svuint32_t e_normal_bits = svlsl_n_u32_x(pg4, svsub_n_u32_x(pg4, e_u32, 1), 23);
|
||||
const svuint32_t e_bits = svsel_u32(e_is_denormal, e_denormal_bits, e_normal_bits);
|
||||
const svfloat16_t d_f16 = svreinterpret_f16_u16(svdup_n_u16(a_ptr[l].d));
|
||||
const svfloat32_t d_f32 = svcvt_f32_f16_x(pg4, d_f16);
|
||||
const svfloat32_t scale = svmul_f32_x(pg4, svreinterpret_f32_u32(e_bits), d_f32);
|
||||
sumf = svmla_f32_m(pg4, sumf,svcvt_f32_s32_m(svdup_n_f32(0.0f), pg4, sumi_cols), scale);
|
||||
}
|
||||
svst1_f32(pg4, res_ptr + x * 4, sumf);
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif // defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
|
||||
|
||||
#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
|
||||
const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);
|
||||
const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
|
||||
@@ -1845,71 +1791,6 @@ void ggml_gemv_q8_0_4x8_q8_0(int n,
|
||||
UNUSED(ncols_interleaved);
|
||||
UNUSED(blocklen);
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
|
||||
// 256-bit SVE: 32 x int8 -> 8 x int32
|
||||
if (ggml_cpu_get_sve_cnt() == QK8_0) {
|
||||
const block_q8_0x4 * GGML_RESTRICT b_ptr = (const block_q8_0x4 *) vx;
|
||||
|
||||
const svbool_t pg = svptrue_b8();
|
||||
const svbool_t pg4w = svptrue_pat_b32(SV_VL4);
|
||||
|
||||
// after svtbl + svcvt the scale lanes are [d0,d0,d1,d1,d2,d2,d3,d3]: lane pair j = row j
|
||||
const svuint16_t d_idx = svreinterpret_u16_u32(svlsr_n_u32_x(pg, svindex_u32(0, 1), 1));
|
||||
|
||||
for (int c = 0; c < nc; c += ncols_interleaved) {
|
||||
const block_q8_0 * GGML_RESTRICT a_ptr = (const block_q8_0 *) vy;
|
||||
|
||||
svfloat32_t acc = svdup_n_f32(0.0f);
|
||||
|
||||
for (int b = 0; b < nb; ++b) {
|
||||
const int8_t * GGML_RESTRICT bq = (const int8_t *) b_ptr->qs;
|
||||
|
||||
uint64_t a_chunk0;
|
||||
uint64_t a_chunk1;
|
||||
uint64_t a_chunk2;
|
||||
uint64_t a_chunk3;
|
||||
uint64_t b_d;
|
||||
|
||||
memcpy(&a_chunk0, a_ptr->qs + 0, sizeof(a_chunk0));
|
||||
memcpy(&a_chunk1, a_ptr->qs + 8, sizeof(a_chunk1));
|
||||
memcpy(&a_chunk2, a_ptr->qs + 16, sizeof(a_chunk2));
|
||||
memcpy(&a_chunk3, a_ptr->qs + 24, sizeof(a_chunk3));
|
||||
memcpy(&b_d, b_ptr->d, sizeof(b_d));
|
||||
|
||||
const svint8_t aq0 = svreinterpret_s8_u64(svdup_n_u64(a_chunk0));
|
||||
const svint8_t aq1 = svreinterpret_s8_u64(svdup_n_u64(a_chunk1));
|
||||
const svint8_t aq2 = svreinterpret_s8_u64(svdup_n_u64(a_chunk2));
|
||||
const svint8_t aq3 = svreinterpret_s8_u64(svdup_n_u64(a_chunk3));
|
||||
|
||||
const svint8_t bq0 = svld1_s8(pg, bq + 0);
|
||||
const svint8_t bq1 = svld1_s8(pg, bq + 32);
|
||||
const svint8_t bq2 = svld1_s8(pg, bq + 64);
|
||||
const svint8_t bq3 = svld1_s8(pg, bq + 96);
|
||||
|
||||
const svint32_t iacc0 = svdot_s32(svdup_n_s32(0), bq0, aq0);
|
||||
const svint32_t iacc1 = svdot_s32(svdup_n_s32(0), bq1, aq1);
|
||||
const svint32_t iacc2 = svdot_s32(svdup_n_s32(0), bq2, aq2);
|
||||
const svint32_t iacc3 = svdot_s32(svdup_n_s32(0), bq3, aq3);
|
||||
|
||||
const svint32_t isum = svadd_s32_x(pg, svadd_s32_x(pg, iacc0, iacc1), svadd_s32_x(pg, iacc2, iacc3));
|
||||
|
||||
const svfloat16_t bd_h = svtbl_f16(svreinterpret_f16_u64(svdup_n_u64(b_d)), d_idx);
|
||||
const svfloat32_t scale = svmul_n_f32_x(pg, svcvt_f32_f16_x(pg, bd_h), GGML_CPU_FP16_TO_FP32(a_ptr->d));
|
||||
|
||||
acc = svmla_f32_x(pg, acc, svcvt_f32_s32_x(pg, isum), scale);
|
||||
|
||||
++a_ptr;
|
||||
++b_ptr;
|
||||
}
|
||||
|
||||
svst1_f32(pg4w, s, svadd_f32_x(pg, svuzp1_f32(acc, acc), svuzp2_f32(acc, acc)));
|
||||
s += ncols_interleaved;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
#endif // defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
|
||||
const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx;
|
||||
|
||||
@@ -4993,7 +4874,6 @@ void ggml_gemm_q6_K_8x8_q8_K(int n,
|
||||
|
||||
constexpr int ncols_interleaved = 8;
|
||||
constexpr int blocklen = 8;
|
||||
constexpr int q8_k_blocklen = 4;
|
||||
|
||||
assert(n % qk == 0);
|
||||
assert(nr % 4 == 0);
|
||||
@@ -5002,195 +4882,9 @@ void ggml_gemm_q6_K_8x8_q8_K(int n,
|
||||
UNUSED(nb);
|
||||
UNUSED(ncols_interleaved);
|
||||
UNUSED(blocklen);
|
||||
UNUSED(q8_k_blocklen);
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
if (svcntb() * 8 == 256) {
|
||||
const svbool_t pg8 = svptrue_b8();
|
||||
const svbool_t pg32 = svptrue_b32();
|
||||
|
||||
const svuint8_t low_nibble = svdup_n_u8(0x0f);
|
||||
const svuint8_t high_lo2 = svdup_n_u8(0x03);
|
||||
const svuint8_t high_hi2 = svdup_n_u8(0x30);
|
||||
const svint8_t q6_bias = svdup_n_s8(32);
|
||||
const svint32_t zero_s32 = svdup_n_s32(0);
|
||||
|
||||
for (int y = 0; y < nr / q8_k_blocklen; ++y) {
|
||||
const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + y * nb;
|
||||
|
||||
for (int x = 0; x < nc / ncols_interleaved; ++x) {
|
||||
const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + x * nb;
|
||||
|
||||
// one accumlator per output row
|
||||
svfloat32_t acc_f32_r0 = svdup_n_f32(0);
|
||||
svfloat32_t acc_f32_r1 = svdup_n_f32(0);
|
||||
svfloat32_t acc_f32_r2 = svdup_n_f32(0);
|
||||
svfloat32_t acc_f32_r3 = svdup_n_f32(0);
|
||||
|
||||
for (int b = 0; b < nb; ++b) {
|
||||
svint32_t acc_c03_r01 = zero_s32;
|
||||
svint32_t acc_c03_r23 = zero_s32;
|
||||
svint32_t acc_c47_r01 = zero_s32;
|
||||
svint32_t acc_c47_r23 = zero_s32;
|
||||
|
||||
for (int half = 0; half < 2; ++half) {
|
||||
const uint8_t * ql_base = q6_ptr[b].ql + half * 512;
|
||||
const uint8_t * qh_base = q6_ptr[b].qh + half * 256;
|
||||
|
||||
for (int sb = 0; sb < QK_K / 64; ++sb) {
|
||||
const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;
|
||||
const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;
|
||||
|
||||
// load 16 bytes (8B of R0, 8B of R1) and repeat in upper half
|
||||
// low lanes will multiply with 16 bytes of C0 and high lanes will multiply with 16 bytes of C1
|
||||
const svint8_t q8_l_r01_k0 = svld1rq_s8(pg8, q8_base_l + 0);
|
||||
const svint8_t q8_l_r01_k1 = svld1rq_s8(pg8, q8_base_l + 32);
|
||||
const svint8_t q8_l_r23_k0 = svld1rq_s8(pg8, q8_base_l + 16);
|
||||
const svint8_t q8_l_r23_k1 = svld1rq_s8(pg8, q8_base_l + 48);
|
||||
|
||||
const svint8_t q8_h_r01_k0 = svld1rq_s8(pg8, q8_base_h + 0);
|
||||
const svint8_t q8_h_r01_k1 = svld1rq_s8(pg8, q8_base_h + 32);
|
||||
const svint8_t q8_h_r23_k0 = svld1rq_s8(pg8, q8_base_h + 16);
|
||||
const svint8_t q8_h_r23_k1 = svld1rq_s8(pg8, q8_base_h + 48);
|
||||
|
||||
const int ql_off = sb * QK_K / 2;
|
||||
const int qh_off = ql_off & 255;
|
||||
|
||||
const int scale_idx_l = half * 8 + sb;
|
||||
const int scale_idx_h = scale_idx_l + 4;
|
||||
|
||||
const svint32_t scales_l_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_l * 8);
|
||||
const svint32_t scales_h_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_h * 8);
|
||||
|
||||
const svint32_t scales_l_c03 = svzip1_s32(scales_l_all, scales_l_all);
|
||||
const svint32_t scales_h_c03 = svzip1_s32(scales_h_all, scales_h_all);
|
||||
|
||||
{
|
||||
svuint8_t q6_ql0_01 = svld1_u8(pg8, ql_base + ql_off);
|
||||
svuint8_t q6_ql1_01 = svld1_u8(pg8, ql_base + ql_off + 64);
|
||||
svuint8_t q6_qh0_01 = svld1_u8(pg8, qh_base + qh_off);
|
||||
svuint8_t q6_qh1_01 = svld1_u8(pg8, qh_base + qh_off + 64);
|
||||
|
||||
if (sb > 1) {
|
||||
q6_qh0_01 = svlsr_n_u8_x(pg8, q6_qh0_01, 2);
|
||||
q6_qh1_01 = svlsr_n_u8_x(pg8, q6_qh1_01, 2);
|
||||
}
|
||||
|
||||
const svuint8_t q6_qh0_01_hi = svand_u8_x(pg8, q6_qh0_01, high_hi2);
|
||||
const svuint8_t q6_qh1_01_hi = svand_u8_x(pg8, q6_qh1_01, high_hi2);
|
||||
|
||||
const svuint8_t q6_l0_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_01, high_lo2), 4));
|
||||
const svuint8_t q6_l1_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_01, high_lo2), 4));
|
||||
const svuint8_t q6_h0_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_01, 4), q6_qh0_01_hi);
|
||||
const svuint8_t q6_h1_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_01, 4), q6_qh1_01_hi);
|
||||
|
||||
const svint8_t q6_l0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_01_bits), q6_bias);
|
||||
const svint8_t q6_l1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_01_bits), q6_bias);
|
||||
const svint8_t q6_h0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_01_bits), q6_bias);
|
||||
const svint8_t q6_h1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_01_bits), q6_bias);
|
||||
|
||||
svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r01_k0);
|
||||
dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_01, q8_l_r01_k1);
|
||||
|
||||
svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r01_k0);
|
||||
dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_01, q8_h_r01_k1);
|
||||
|
||||
svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r23_k0);
|
||||
dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_01, q8_l_r23_k1);
|
||||
|
||||
svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r23_k0);
|
||||
dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_01, q8_h_r23_k1);
|
||||
|
||||
acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_l_r01, scales_l_c03);
|
||||
acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_h_r01, scales_h_c03);
|
||||
acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_l_r23, scales_l_c03);
|
||||
acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_h_r23, scales_h_c03);
|
||||
}
|
||||
|
||||
const svint32_t scales_l_c47 = svzip2_s32(scales_l_all, scales_l_all);
|
||||
const svint32_t scales_h_c47 = svzip2_s32(scales_h_all, scales_h_all);
|
||||
|
||||
{
|
||||
svuint8_t q6_ql0_23 = svld1_u8(pg8, ql_base + ql_off + 32);
|
||||
svuint8_t q6_ql1_23 = svld1_u8(pg8, ql_base + ql_off + 96);
|
||||
svuint8_t q6_qh0_23 = svld1_u8(pg8, qh_base + qh_off + 32);
|
||||
svuint8_t q6_qh1_23 = svld1_u8(pg8, qh_base + qh_off + 96);
|
||||
|
||||
if (sb > 1) {
|
||||
q6_qh0_23 = svlsr_n_u8_x(pg8, q6_qh0_23, 2);
|
||||
q6_qh1_23 = svlsr_n_u8_x(pg8, q6_qh1_23, 2);
|
||||
}
|
||||
|
||||
const svuint8_t q6_qh0_23_hi = svand_u8_x(pg8, q6_qh0_23, high_hi2);
|
||||
const svuint8_t q6_qh1_23_hi = svand_u8_x(pg8, q6_qh1_23, high_hi2);
|
||||
|
||||
const svuint8_t q6_l0_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_23, high_lo2), 4));
|
||||
const svuint8_t q6_l1_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_23, high_lo2), 4));
|
||||
const svuint8_t q6_h0_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_23, 4), q6_qh0_23_hi);
|
||||
const svuint8_t q6_h1_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_23, 4), q6_qh1_23_hi);
|
||||
|
||||
const svint8_t q6_l0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_23_bits), q6_bias);
|
||||
const svint8_t q6_l1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_23_bits), q6_bias);
|
||||
const svint8_t q6_h0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_23_bits), q6_bias);
|
||||
const svint8_t q6_h1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_23_bits), q6_bias);
|
||||
|
||||
svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r01_k0);
|
||||
dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_23, q8_l_r01_k1);
|
||||
|
||||
svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r01_k0);
|
||||
dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_23, q8_h_r01_k1);
|
||||
|
||||
svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r23_k0);
|
||||
dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_23, q8_l_r23_k1);
|
||||
|
||||
svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r23_k0);
|
||||
dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_23, q8_h_r23_k1);
|
||||
|
||||
acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_l_r01, scales_l_c47);
|
||||
acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_h_r01, scales_h_c47);
|
||||
acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_l_r23, scales_l_c47);
|
||||
acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_h_r23, scales_h_c47);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// acc_c03_r01 -> c0-r0,c0-r1,c1-r0,c1-r1,c2-r0,c2-r1,c3-r0,c3-r1
|
||||
// acc_c47_r01 -> c4-r0,c4-r1,c5-r0,c5-r1,c6-r0,c6-r1,c7-r0,c7-r1
|
||||
// svuzp1_s32 picks even lanes -> c0-r0,c1-r0,c2-r0,c3-r0,c4-r0,c5-r0,c6-r0,c7-r0
|
||||
// similarly svuzp2_s32 picks remaining values from odd lanes
|
||||
const svint32_t dot_r0 = svuzp1_s32(acc_c03_r01, acc_c47_r01);
|
||||
const svint32_t dot_r1 = svuzp2_s32(acc_c03_r01, acc_c47_r01);
|
||||
const svint32_t dot_r2 = svuzp1_s32(acc_c03_r23, acc_c47_r23);
|
||||
const svint32_t dot_r3 = svuzp2_s32(acc_c03_r23, acc_c47_r23);
|
||||
|
||||
const svuint32_t q6_d_bits = svld1uh_u32(pg32, (const uint16_t *) q6_ptr[b].d);
|
||||
const svfloat32_t q6_d_f32 = svcvt_f32_f16_x(pg32, svreinterpret_f16_u32(q6_d_bits));
|
||||
|
||||
const svfloat32_t scale_r0 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[0]);
|
||||
const svfloat32_t scale_r1 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[1]);
|
||||
const svfloat32_t scale_r2 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[2]);
|
||||
const svfloat32_t scale_r3 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[3]);
|
||||
|
||||
acc_f32_r0 = svmla_f32_x(pg32, acc_f32_r0, svcvt_f32_s32_x(pg32, dot_r0), scale_r0);
|
||||
acc_f32_r1 = svmla_f32_x(pg32, acc_f32_r1, svcvt_f32_s32_x(pg32, dot_r1), scale_r1);
|
||||
acc_f32_r2 = svmla_f32_x(pg32, acc_f32_r2, svcvt_f32_s32_x(pg32, dot_r2), scale_r2);
|
||||
acc_f32_r3 = svmla_f32_x(pg32, acc_f32_r3, svcvt_f32_s32_x(pg32, dot_r3), scale_r3);
|
||||
}
|
||||
|
||||
const int row = y * q8_k_blocklen;
|
||||
const int col = x * ncols_interleaved;
|
||||
|
||||
svst1_f32(pg32, s + (row + 0) * bs + col, acc_f32_r0);
|
||||
svst1_f32(pg32, s + (row + 1) * bs + col, acc_f32_r1);
|
||||
svst1_f32(pg32, s + (row + 2) * bs + col, acc_f32_r2);
|
||||
svst1_f32(pg32, s + (row + 3) * bs + col, acc_f32_r3);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif // SVE compile-time end
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
constexpr int q8_k_blocklen = 4;
|
||||
const uint8x16_t m4b = vdupq_n_u8(0x0f);
|
||||
const uint8x16_t mask_lo = vdupq_n_u8(0x03);
|
||||
const uint8x16_t mask_hi = vdupq_n_u8(0x30);
|
||||
|
||||
@@ -43,14 +43,6 @@ static inline float f32_to_f32(float x) {
|
||||
return x;
|
||||
}
|
||||
|
||||
static inline float f8_e4m3_to_f32(ggml_fp8_e4m3_t x) {
|
||||
return ggml_f8_e4m3_to_fp32(x.bits);
|
||||
}
|
||||
|
||||
static inline ggml_fp8_e4m3_t f32_to_f8_e4m3(float x) {
|
||||
return { ggml_fp32_to_f8_e4m3(x) };
|
||||
}
|
||||
|
||||
// TODO - merge this into the traits table, after using row-based conversions
|
||||
template <class T>
|
||||
struct type_conversion_table;
|
||||
@@ -73,12 +65,6 @@ struct type_conversion_table<ggml_bf16_t> {
|
||||
static constexpr ggml_bf16_t (*from_f32)(float) = f32_to_bf16;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct type_conversion_table<ggml_fp8_e4m3_t> {
|
||||
static constexpr float (*to_f32)(ggml_fp8_e4m3_t) = f8_e4m3_to_f32;
|
||||
static constexpr ggml_fp8_e4m3_t (*from_f32)(float) = f32_to_f8_e4m3;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct type_conversion_table<int32_t> {
|
||||
static constexpr float (*to_f32)(int32_t) = i32_to_f32;
|
||||
|
||||
@@ -86,9 +86,6 @@ float ggml_table_f32_e8m0_half[1 << 8];
|
||||
// precomputed f32 table for ue4m3 (1 KB) (simd-mappings.h)
|
||||
float ggml_table_f32_ue4m3[1 << 8];
|
||||
|
||||
// Signed FP8 needs its own table: UE4M3 ignores the sign, halves values for NVFP4, and maps NaN to zero.
|
||||
float ggml_table_f32_f8_e4m3[1 << 8];
|
||||
|
||||
#if defined(__ARM_ARCH)
|
||||
struct ggml_arm_arch_features_type {
|
||||
int sve_cnt;
|
||||
@@ -299,12 +296,6 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
|
||||
.vec_dot_type = GGML_TYPE_Q8_0,
|
||||
.nrows = 1,
|
||||
},
|
||||
[GGML_TYPE_F8_E4M3] = {
|
||||
.from_float = quantize_row_f8_e4m3,
|
||||
.vec_dot = ggml_vec_dot_f8_e4m3_f32,
|
||||
.vec_dot_type = GGML_TYPE_F32,
|
||||
.nrows = 1,
|
||||
},
|
||||
[GGML_TYPE_Q2_K] = {
|
||||
.from_float = quantize_row_q2_K,
|
||||
.vec_dot = ggml_vec_dot_q2_K_q8_K,
|
||||
@@ -2885,7 +2876,7 @@ struct ggml_cplan ggml_graph_plan(
|
||||
case GGML_OP_ADD_ID:
|
||||
case GGML_OP_ADD1:
|
||||
{
|
||||
if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3) {
|
||||
if (ggml_is_quantized(node->src[0]->type)) {
|
||||
cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
|
||||
}
|
||||
} break;
|
||||
@@ -2933,7 +2924,7 @@ struct ggml_cplan ggml_graph_plan(
|
||||
} break;
|
||||
case GGML_OP_OUT_PROD:
|
||||
{
|
||||
if (ggml_is_quantized(node->src[0]->type) || node->src[0]->type == GGML_TYPE_F8_E4M3 ||
|
||||
if (ggml_is_quantized(node->src[0]->type) ||
|
||||
node->src[0]->type == GGML_TYPE_F16) {
|
||||
cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
|
||||
}
|
||||
@@ -3917,7 +3908,6 @@ void ggml_cpu_init(void) {
|
||||
// initialize UE4M3 table (256 entries)
|
||||
for (int i = 0; i < (1 << 8); ++i) {
|
||||
ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32(i);
|
||||
ggml_table_f32_f8_e4m3[i] = ggml_f8_e4m3_to_fp32(i);
|
||||
}
|
||||
|
||||
const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
|
||||
|
||||
+14
-30
@@ -537,36 +537,26 @@ void ggml_compute_forward_dup(
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
/**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_fp16_t, float >(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp8_e4m3_t>(params, dst);
|
||||
/**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_fp16_t, float >(params, dst);
|
||||
else ggml_compute_forward_dup_to_q<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
/**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_bf16_t, float >(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp8_e4m3_t>(params, dst);
|
||||
/**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_bf16_t, float >(params, dst);
|
||||
else ggml_compute_forward_dup_to_q<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
/**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<float, float >(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F8_E4M3) ggml_compute_forward_dup_flt<float, ggml_fp8_e4m3_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_I32) ggml_compute_forward_dup_flt<float, int32_t >(params, dst);
|
||||
/**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<float, float >(params, dst);
|
||||
else if (dst->type == GGML_TYPE_I32) ggml_compute_forward_dup_flt<float, int32_t >(params, dst);
|
||||
else ggml_compute_forward_dup_to_q<float>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
{
|
||||
/**/ if (dst->type == GGML_TYPE_F16) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, ggml_fp16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, ggml_bf16_t>(params, dst);
|
||||
else if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<ggml_fp8_e4m3_t, float >(params, dst);
|
||||
else GGML_ABORT("not implemented");
|
||||
} break;
|
||||
case GGML_TYPE_I32:
|
||||
{
|
||||
if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<int32_t, float>(params, dst);
|
||||
@@ -615,9 +605,8 @@ static void ggml_compute_forward_add_q_f32(
|
||||
GGML_ASSERT(nb1 <= nb2);
|
||||
GGML_ASSERT(nb2 <= nb3);
|
||||
|
||||
GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3);
|
||||
GGML_ASSERT(ggml_is_quantized(src0->type));
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(ne00 % ggml_blck_size(type) == 0);
|
||||
|
||||
// rows per thread
|
||||
const int dr = (nr + nth - 1)/nth;
|
||||
@@ -647,6 +636,8 @@ static void ggml_compute_forward_add_q_f32(
|
||||
float * src1_row = (float *)((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13));
|
||||
void * dst_row = (void *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3));
|
||||
|
||||
assert(ne00 % 32 == 0);
|
||||
|
||||
// unquantize row from src0 to temp buffer
|
||||
dequantize_row_q(src0_row, wdata, ne00);
|
||||
// add src1
|
||||
@@ -682,7 +673,6 @@ void ggml_compute_forward_add(
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
@@ -965,7 +955,7 @@ static void ggml_compute_forward_add1_q_f32(
|
||||
GGML_ASSERT(nb1 <= nb2);
|
||||
GGML_ASSERT(nb2 <= nb3);
|
||||
|
||||
GGML_ASSERT(ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3);
|
||||
GGML_ASSERT(ggml_is_quantized(src0->type));
|
||||
GGML_ASSERT(dst->type == src0->type);
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||
|
||||
@@ -1140,7 +1130,6 @@ void ggml_compute_forward_add1(
|
||||
case GGML_TYPE_Q8_1:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
@@ -1272,7 +1261,6 @@ void ggml_compute_forward_acc(
|
||||
case GGML_TYPE_Q8_1:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
@@ -4816,7 +4804,6 @@ void ggml_compute_forward_out_prod(
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
@@ -5142,7 +5129,6 @@ void ggml_compute_forward_set(
|
||||
case GGML_TYPE_Q8_1:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
@@ -5368,7 +5354,6 @@ void ggml_compute_forward_get_rows(
|
||||
case GGML_TYPE_Q8_1:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
@@ -6129,7 +6114,6 @@ void ggml_compute_forward_clamp(
|
||||
case GGML_TYPE_Q8_1:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_F8_E4M3:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
#include "ggml-cpu-impl.h"
|
||||
#include "simd-mappings.h"
|
||||
#include "vec.h"
|
||||
#include "ggml-quants.h"
|
||||
#include "quants.h"
|
||||
|
||||
@@ -63,10 +62,6 @@ void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i
|
||||
quantize_row_nvfp4_ref(x, y, k);
|
||||
}
|
||||
|
||||
void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
|
||||
quantize_row_f8_e4m3_ref(x, y, k);
|
||||
}
|
||||
|
||||
//
|
||||
// 2-6 bit quantization in super-blocks
|
||||
//
|
||||
@@ -367,49 +362,6 @@ void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs,
|
||||
*s = sumf;
|
||||
}
|
||||
|
||||
void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
||||
assert(nrc == 1);
|
||||
UNUSED(nrc);
|
||||
UNUSED(bx);
|
||||
UNUSED(by);
|
||||
UNUSED(bs);
|
||||
|
||||
const ggml_fp8_e4m3_t * GGML_RESTRICT x = vx;
|
||||
const float * GGML_RESTRICT y = vy;
|
||||
|
||||
float sumf = 0.0f;
|
||||
int pos = 0;
|
||||
#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)
|
||||
GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };
|
||||
const int np = n - n % GGML_F32_STEP;
|
||||
for (; pos < np; pos += GGML_F32_STEP) {
|
||||
for (int j = 0; j < GGML_F32_ARR; ++j) {
|
||||
const int offset = pos + j*GGML_F32_EPR;
|
||||
GGML_F32_VEC ax;
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
const __m128i codes = _mm_loadu_si128((const __m128i *) (x + offset));
|
||||
ax = _mm512_i32gather_ps(_mm512_cvtepu8_epi32(codes), ggml_table_f32_f8_e4m3, sizeof(float));
|
||||
#elif defined(__AVX2__) && !defined(__AVX512F__)
|
||||
const __m128i codes = _mm_loadl_epi64((const __m128i *) (x + offset));
|
||||
ax = _mm256_i32gather_ps(ggml_table_f32_f8_e4m3, _mm256_cvtepu8_epi32(codes), sizeof(float));
|
||||
#else
|
||||
float values[GGML_F32_EPR];
|
||||
for (int k = 0; k < GGML_F32_EPR; ++k) {
|
||||
values[k] = ggml_table_f32_f8_e4m3[x[offset + k].bits];
|
||||
}
|
||||
ax = GGML_F32_VEC_LOAD(values);
|
||||
#endif
|
||||
sum[j] = GGML_F32_VEC_FMA(sum[j], ax, GGML_F32_VEC_LOAD(y + offset));
|
||||
}
|
||||
}
|
||||
GGML_F32_VEC_REDUCE(sumf, sum);
|
||||
#endif
|
||||
for (; pos < n; ++pos) {
|
||||
sumf += ggml_table_f32_f8_e4m3[x[pos].bits] * y[pos];
|
||||
}
|
||||
*s = sumf;
|
||||
}
|
||||
|
||||
void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
||||
const int qk = QK8_0;
|
||||
const int nb = n / qk;
|
||||
|
||||
@@ -23,7 +23,6 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in
|
||||
|
||||
void quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||
void quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||
void quantize_row_f8_e4m3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||
|
||||
void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||
void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||
@@ -41,7 +40,6 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y,
|
||||
// Dot product
|
||||
void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
|
||||
@@ -127,8 +127,6 @@ extern float ggml_table_f32_e8m0_half[1 << 8];
|
||||
// defined in ggml-cpu.c, initialized in ggml_cpu_init()
|
||||
extern float ggml_table_f32_ue4m3[1 << 8];
|
||||
|
||||
extern float ggml_table_f32_f8_e4m3[1 << 8];
|
||||
|
||||
// Use lookup table for E8M0 on x86 (faster than bit manipulation)
|
||||
#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
|
||||
#define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)]
|
||||
|
||||
@@ -1489,9 +1489,6 @@ static const ggml_backend_buffer_i ggml_backend_riscv64_spacemit_buffer_i = {
|
||||
/* .cpy_tensor = */ nullptr,
|
||||
/* .clear = */ ggml_backend_riscv64_spacemit_buffer_clear,
|
||||
/* .reset = */ nullptr,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
static const char * ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
|
||||
|
||||
@@ -82,7 +82,6 @@
|
||||
#define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a
|
||||
#define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA
|
||||
#define GGML_CUDA_CC_RDNA3_5 (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops.
|
||||
#define GGML_CUDA_CC_RDNA4_M (GGML_CUDA_CC_OFFSET_AMD + 0x1170) // gfx117X
|
||||
#define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000
|
||||
|
||||
#define GGML_CUDA_CC_IS_AMD(cc) (cc >= GGML_CUDA_CC_OFFSET_AMD)
|
||||
@@ -90,9 +89,8 @@
|
||||
#define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)
|
||||
#define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)
|
||||
#define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5)
|
||||
#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4_M)
|
||||
#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4)
|
||||
#define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc))
|
||||
#define GGML_CUDA_CC_IS_RDNA4_M(cc) (cc >= GGML_CUDA_CC_RDNA4_M && cc < GGML_CUDA_CC_RDNA4)
|
||||
#define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4)
|
||||
#define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C)
|
||||
#define GGML_CUDA_CC_IS_GCN(cc) ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc))
|
||||
@@ -279,17 +277,9 @@ static const char * cu_get_error_str(CUresult err) {
|
||||
#define AMD_MFMA_AVAILABLE
|
||||
#endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)
|
||||
|
||||
#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA4_M))
|
||||
#define AMD_WMMA_128B_AVAILABLE
|
||||
#endif // defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA4_M))
|
||||
|
||||
#if defined(GGML_USE_HIP) && defined(RDNA3)
|
||||
#define AMD_WMMA_256B_AVAILABLE
|
||||
#endif // defined(GGML_USE_HIP) && defined(RDNA3)
|
||||
|
||||
#if defined(AMD_WMMA_128B_AVAILABLE) || defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3))
|
||||
#define AMD_WMMA_AVAILABLE
|
||||
#endif // defined(AMD_WMMA_128B_AVAILABLE) || defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#endif // defined(GGML_USE_HIP) && defined(RDNA4)
|
||||
|
||||
// The Volta instructions are in principle available on Turing or newer but they are effectively unusable:
|
||||
#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA
|
||||
@@ -336,7 +326,7 @@ static bool fast_fp16_hardware_available(const int cc) {
|
||||
// To be used for feature selection of external libraries, e.g. cuBLAS.
|
||||
static bool fp16_mma_hardware_available(const int cc) {
|
||||
return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) ||
|
||||
GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||
|
||||
GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||
|
||||
(GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);
|
||||
}
|
||||
|
||||
@@ -365,7 +355,7 @@ static bool amd_mfma_available(const int cc) {
|
||||
}
|
||||
|
||||
static bool amd_wmma_available(const int cc) {
|
||||
return GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc);
|
||||
return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc));
|
||||
}
|
||||
|
||||
static bool volta_mma_available(const int cc) {
|
||||
@@ -723,7 +713,7 @@ static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, i
|
||||
#if defined(GGML_USE_HIP)
|
||||
#if defined(CDNA) || defined(RDNA2) || defined(__gfx906__)
|
||||
c = __builtin_amdgcn_sdot4(a, b, c, false);
|
||||
#elif defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4)
|
||||
#elif defined(RDNA3) || defined(RDNA4)
|
||||
c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);
|
||||
#elif defined(RDNA1) || defined(__gfx900__)
|
||||
int tmp1;
|
||||
@@ -768,9 +758,9 @@ static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v
|
||||
acc += v.y*u.y;
|
||||
}
|
||||
|
||||
#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
|
||||
#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
|
||||
#define V_DOT2_F32_F16_AVAILABLE
|
||||
#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4_M) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
|
||||
#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))
|
||||
|
||||
static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) {
|
||||
#ifdef V_DOT2_F32_F16_AVAILABLE
|
||||
|
||||
@@ -1255,7 +1255,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_B_KQ Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)];
|
||||
#if defined(TURING_MMA_AVAILABLE)
|
||||
T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)];
|
||||
#elif defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
|
||||
T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)];
|
||||
#elif defined(AMD_MFMA_AVAILABLE)
|
||||
T_C_VKQ VKQ_C[ DV/T_C_VKQ::J];
|
||||
@@ -1866,7 +1866,7 @@ static __global__ void flash_attn_ext_f16(
|
||||
#endif // defined(AMD_WMMA_AVAILABLE)
|
||||
|
||||
#if defined(AMD_MFMA_AVAILABLE)
|
||||
if (ncols1*ncols2 < 16 || (DKQ > 256 && ncols1*ncols2 < 32 && !use_sparse)) {
|
||||
if (ncols1*ncols2 < 16 || (DKQ > 256 && ncols1*ncols2 < 32)) {
|
||||
NO_DEVICE_CODE;
|
||||
return;
|
||||
}
|
||||
@@ -2054,7 +2054,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
|
||||
bool use_sparse = false;
|
||||
if (logit_softcap == 0.0f) {
|
||||
constexpr bool use_logit_softcap = false;
|
||||
#if !defined(GGML_USE_MUSA)
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2)) {
|
||||
if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, ncols1, ncols2)) {
|
||||
constexpr bool use_sparse_kernel = true;
|
||||
@@ -2077,7 +2077,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
|
||||
}
|
||||
}
|
||||
} else
|
||||
#endif // !defined(GGML_USE_MUSA)
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
{
|
||||
constexpr bool use_sparse_kernel = false;
|
||||
fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;
|
||||
|
||||
+25
-49
@@ -5,8 +5,7 @@
|
||||
#include "fattn-vec.cuh"
|
||||
#include "fattn.cuh"
|
||||
|
||||
#if !defined(GGML_USE_MUSA)
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
// one list per group of ncols1 queries: a column is selected if any query of the group can see it
|
||||
template <int ncols1, bool oob>
|
||||
__launch_bounds__(256, 1)
|
||||
@@ -15,20 +14,10 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
const int n_kv_max, const int64_t s31, const int64_t s33) {
|
||||
ggml_cuda_pdl_sync();
|
||||
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int values_per_lane = 8;
|
||||
using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
|
||||
constexpr uint64_t ballot_mask = ~(lane_mask_t)0;
|
||||
const auto popc = [](const lane_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
return __popc(mask);
|
||||
}
|
||||
};
|
||||
const int tid = threadIdx.x;
|
||||
const int warp = tid / warp_size;
|
||||
const int lane = tid % warp_size;
|
||||
const int warp = tid / WARP_SIZE;
|
||||
const int lane = tid % WARP_SIZE;
|
||||
const int sequence = blockIdx.y;
|
||||
const int group = blockIdx.x;
|
||||
|
||||
@@ -38,7 +27,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
const half * mask = mask_ptr + sequence*s33 + q0*s31;
|
||||
int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + group)*n_kv_max;
|
||||
|
||||
__shared__ int warp_offsets[256/warp_size];
|
||||
__shared__ int warp_offsets[256/WARP_SIZE];
|
||||
__shared__ int row_count;
|
||||
__shared__ int chunk_count;
|
||||
|
||||
@@ -48,11 +37,11 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
__syncthreads();
|
||||
|
||||
for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
|
||||
lane_mask_t selected_warp[values_per_lane];
|
||||
uint32_t selected_warp[values_per_lane];
|
||||
int warp_count = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
|
||||
const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;
|
||||
bool selected = false;
|
||||
if (i < ne30) {
|
||||
#pragma unroll
|
||||
@@ -60,8 +49,8 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
|
||||
}
|
||||
}
|
||||
selected_warp[item] = (lane_mask_t)__ballot_sync(ballot_mask, (int) selected);
|
||||
warp_count += popc(selected_warp[item]);
|
||||
selected_warp[item] = __ballot_sync(0xFFFFFFFF, selected);
|
||||
warp_count += __popc(selected_warp[item]);
|
||||
}
|
||||
|
||||
if (lane == 0) {
|
||||
@@ -72,7 +61,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
if (tid == 0) {
|
||||
int offset = 0;
|
||||
#pragma unroll
|
||||
for (int iw = 0; iw < 256/warp_size; ++iw) {
|
||||
for (int iw = 0; iw < 256/WARP_SIZE; ++iw) {
|
||||
const int count = warp_offsets[iw];
|
||||
warp_offsets[iw] = offset;
|
||||
offset += count;
|
||||
@@ -81,16 +70,16 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1;
|
||||
const uint32_t lane_mask = lane == 0 ? 0 : (1u << lane) - 1;
|
||||
int warp_item_offset = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + __popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & (uint32_t(1) << lane)) && dst < n_kv_max) {
|
||||
indices[dst] = i;
|
||||
}
|
||||
warp_item_offset += popc(selected_warp[item]);
|
||||
warp_item_offset += __popc(selected_warp[item]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
@@ -112,13 +101,13 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
// the dependent grid reads indices, signal once the row is complete
|
||||
ggml_cuda_pdl_lc();
|
||||
}
|
||||
#endif // !defined(GGML_USE_MUSA)
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
void ggml_cuda_flash_attn_ext_compact_mask(
|
||||
const ggml_tensor * mask, int32_t * indices, int32_t * counts, int32_t n_queries, int32_t ncols1, int32_t n_kv_max, cudaStream_t stream) {
|
||||
#ifdef GGML_USE_MUSA
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED_VARS(mask, indices, counts, n_queries, ncols1, n_kv_max, stream);
|
||||
GGML_ABORT("sparse flash attention is not supported on MUSA");
|
||||
GGML_ABORT("sparse flash attention is only supported on NVIDIA CUDA");
|
||||
#else
|
||||
const int64_t s31 = mask->nb[1] / sizeof(half);
|
||||
const int64_t s33 = mask->nb[3] / sizeof(half);
|
||||
@@ -133,17 +122,16 @@ void ggml_cuda_flash_attn_ext_compact_mask(
|
||||
ggml_cuda_kernel_launch(kernel, launch_params,
|
||||
(const half *) mask->data, indices, counts, int(mask->ne[0]), n_queries, n_kv_max, s31, s33);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
#endif // !defined(GGML_USE_MUSA)
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
}
|
||||
|
||||
bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2) {
|
||||
#ifdef GGML_USE_MUSA
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED_VARS(cc, dst, ncols1, ncols2);
|
||||
return false;
|
||||
#else
|
||||
const ggml_tensor * Q = dst->src[0];
|
||||
const ggml_tensor * K = dst->src[1];
|
||||
const ggml_tensor * V = dst->src[2];
|
||||
const ggml_tensor * mask = dst->src[3];
|
||||
|
||||
float max_bias = 0.0f;
|
||||
@@ -151,21 +139,16 @@ bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_
|
||||
memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
|
||||
memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float));
|
||||
|
||||
// On CDNA only the DKQ=576 kernel variant with ncols2=16 is compiled with sparse support.
|
||||
if (amd_mfma_available(cc) && !ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(int(Q->ne[0]), int(V->ne[0]), 1, 16)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4);
|
||||
|
||||
// the dense kernel handles up to 64/ncols2 queries per K/V pass, the single-query gather has to beat that
|
||||
const int64_t n_gather = (ncols1 == 1 ? std::min<int64_t>(Q->ne[1], 64/ncols2) : ncols1) * (int64_t) n_kv_max;
|
||||
|
||||
return ((GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc)) || amd_mfma_available(cc)) &&
|
||||
return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) &&
|
||||
mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f &&
|
||||
mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 &&
|
||||
K->ne[1] >= std::max<int64_t>(4096, GGML_CUDA_CC_IS_NVIDIA(cc) ? 2*n_gather : 2LL*n_kv_max);
|
||||
#endif // !defined(GGML_USE_MUSA)
|
||||
K->ne[1] >= std::max<int64_t>(4096, 2*n_gather);
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
}
|
||||
|
||||
template <int DKQ, int DV, int ncols2>
|
||||
@@ -173,7 +156,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con
|
||||
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
|
||||
const ggml_tensor * Q = dst->src[0];
|
||||
|
||||
#if !defined(GGML_USE_MUSA)
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 1, ncols2)) {
|
||||
// a sparse variant at the full tile width gathers the union of its queries once, prefer it for large batches
|
||||
constexpr bool has_wide_sparse = ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 64/ncols2, ncols2);
|
||||
@@ -182,7 +165,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // !defined(GGML_USE_MUSA)
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
if constexpr (ncols2 <= 8) {
|
||||
if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) {
|
||||
@@ -382,9 +365,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
|
||||
|
||||
GGML_ASSERT(Q->ne[2] % K->ne[2] == 0);
|
||||
const int gqa_ratio = Q->ne[2] / K->ne[2];
|
||||
if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
} else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
|
||||
if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
|
||||
if (cc >= GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (Q->ne[1] <= 8) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
@@ -698,11 +679,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
return BEST_FATTN_KERNEL_MMA_F16;
|
||||
}
|
||||
|
||||
// On CDNA, the sparse MMA kernel is used for DKQ=576 regardless of the head size heuristics.
|
||||
if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
|
||||
return BEST_FATTN_KERNEL_MMA_F16;
|
||||
}
|
||||
|
||||
// AMD MFMA needs a certain minimum batch size to outscale the tile kernel for large head sizes.
|
||||
if (amd_mfma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
|
||||
if ((Q->ne[0] <= 64 && Q->ne[1] * gqa_ratio_eff > 8)) {
|
||||
|
||||
@@ -861,46 +861,6 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
|
||||
static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
||||
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
CUDA_CHECK(cudaMemcpyAsync(
|
||||
(char *) tensors[i]->data + offsets[i],
|
||||
data[i],
|
||||
sizes[i],
|
||||
cudaMemcpyHostToDevice,
|
||||
cudaStreamPerThread));
|
||||
}
|
||||
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
|
||||
static void ggml_backend_cuda_buffer_get_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
||||
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
CUDA_CHECK(cudaMemcpyAsync(
|
||||
data[i],
|
||||
(const char *) tensors[i]->data + offsets[i],
|
||||
sizes[i],
|
||||
cudaMemcpyDeviceToHost,
|
||||
cudaStreamPerThread));
|
||||
}
|
||||
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
|
||||
static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
|
||||
if (ggml_backend_buffer_is_cuda(src->buffer)) {
|
||||
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
|
||||
@@ -926,54 +886,6 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
|
||||
GGML_UNUSED(buffer);
|
||||
}
|
||||
|
||||
static bool ggml_backend_cuda_buffer_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) {
|
||||
ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
|
||||
if (!ggml_backend_buffer_is_cuda(src_buf)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef GGML_CUDA_NO_PEER_COPY
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
|
||||
ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
|
||||
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src_buf->context;
|
||||
ggml_backend_cuda_buffer_context * dst_ctx_i = (ggml_backend_cuda_buffer_context *)dst_buf->context;
|
||||
const int src_physical = ggml_cuda_get_physical_device(src_ctx->device);
|
||||
const int dst_physical = ggml_cuda_get_physical_device(dst_ctx_i->device);
|
||||
if (src_physical != dst_physical) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
ggml_cuda_set_device(dst_ctx->device);
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
|
||||
ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
|
||||
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src_buf->context;
|
||||
ggml_backend_cuda_buffer_context * dst_ctx_i = (ggml_backend_cuda_buffer_context *)dst_buf->context;
|
||||
const int src_physical = ggml_cuda_get_physical_device(src_ctx->device);
|
||||
const int dst_physical = ggml_cuda_get_physical_device(dst_ctx_i->device);
|
||||
if (src_physical == dst_physical) {
|
||||
CUDA_CHECK(cudaMemcpyAsync(dsts[i]->data, srcs[i]->data, ggml_nbytes(srcs[i]), cudaMemcpyDeviceToDevice, cudaStreamPerThread));
|
||||
} else {
|
||||
#ifdef GGML_CUDA_NO_PEER_COPY
|
||||
GGML_ABORT("peer copy not supported");
|
||||
#else
|
||||
CUDA_CHECK(cudaMemcpyPeerAsync(dsts[i]->data, dst_physical, srcs[i]->data, src_physical, ggml_nbytes(srcs[i]), cudaStreamPerThread));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
return true;
|
||||
}
|
||||
|
||||
static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
||||
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
||||
|
||||
@@ -994,9 +906,6 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
|
||||
/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_cuda_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ ggml_backend_cuda_buffer_set_tensor_n,
|
||||
/* .get_tensor_n = */ ggml_backend_cuda_buffer_get_tensor_n,
|
||||
/* .cpy_tensor_n = */ ggml_backend_cuda_buffer_cpy_tensor_n,
|
||||
};
|
||||
|
||||
// cuda buffer type
|
||||
@@ -1656,7 +1565,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
|
||||
if (compute_type == GGML_TYPE_F16) {
|
||||
prefer_f32_output = cc == GGML_CUDA_CC_VOLTA || GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_CDNA(cc);
|
||||
} else if (compute_type == GGML_TYPE_BF16) {
|
||||
prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_RDNA4_M(cc) && !GGML_CUDA_CC_IS_CDNA(cc);
|
||||
prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_CDNA(cc);
|
||||
}
|
||||
|
||||
if (prefer_f32_output) {
|
||||
@@ -2053,7 +1962,7 @@ static bool ggml_cuda_mul_mat_q_fusion_matches(const ggml_tensor * up, const ggm
|
||||
const ggml_tensor * x_up = up->src[0];
|
||||
const ggml_tensor * y = up->src[1];
|
||||
|
||||
if ((up->op != GGML_OP_MUL_MAT && up->op != GGML_OP_MUL_MAT_ID) || !ggml_cuda_mmq_fusion_supported(x_up->type) || y->type != GGML_TYPE_F32) {
|
||||
if (up->op != GGML_OP_MUL_MAT || !ggml_cuda_mmq_fusion_supported(x_up->type) || y->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -2095,18 +2004,12 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t
|
||||
const ggml_tensor * x_up = up->src[0];
|
||||
const ggml_tensor * y = up->src[1];
|
||||
const int cc = ggml_cuda_info().devices[device].cc;
|
||||
|
||||
// For MUL_MAT_ID: same kernel choice as ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
|
||||
const bool is_id = up->op == GGML_OP_MUL_MAT_ID;
|
||||
const int64_t ncols = is_id ? y->ne[2] : y->ne[1];
|
||||
const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3];
|
||||
const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols);
|
||||
if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) {
|
||||
if (ggml_cuda_should_use_mmvq(x_up->type, cc, y->ne[1]) || !ggml_cuda_should_use_mmq(x_up->type, cc, y->ne[1], 0)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// The fused kernel needs one block per tile (no stream-k split), so only fuse if the tiling is efficient.
|
||||
return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], ncols, nchannels, device);
|
||||
return ggml_cuda_mmq_fusion_is_efficient(x_up->type, x_up->ne[1], y->ne[1], y->ne[2]*y->ne[3], device);
|
||||
}
|
||||
|
||||
static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
|
||||
+31
-31
@@ -76,11 +76,11 @@ namespace ggml_cuda_mma {
|
||||
// For the A/C matrices this means I major == row major, J major == column major.
|
||||
// For the B matrix this means I major == column major, J major == row major.
|
||||
// MIRRORED == Each data value is held exactly once per thread subgroup.
|
||||
DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for WMMA128 and CDNA.
|
||||
DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and WMMA128, int and float matrix C for WMMA256.
|
||||
DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for RDNA4 and CDNA.
|
||||
DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3.
|
||||
DATA_LAYOUT_I_MAJOR_MIRRORED = 20, // Volta, matrix A&B for RDNA3.
|
||||
DATA_LAYOUT_J_MAJOR_MIRRORED = 30,
|
||||
DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (WMMA128/CDNA), convert to float to unscramble.
|
||||
DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble.
|
||||
};
|
||||
// Implemented mma combinations are:
|
||||
// - (I_MAJOR, I_MAJOR) -> I_MAJOR
|
||||
@@ -88,11 +88,11 @@ namespace ggml_cuda_mma {
|
||||
// - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR
|
||||
|
||||
static constexpr __device__ data_layout get_input_data_layout() {
|
||||
#if defined(AMD_WMMA_256B_AVAILABLE) || defined(VOLTA_MMA_AVAILABLE)
|
||||
#if defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)
|
||||
return DATA_LAYOUT_I_MAJOR_MIRRORED;
|
||||
#else
|
||||
return DATA_LAYOUT_I_MAJOR;
|
||||
#endif // defined(AMD_WMMA_256B_AVAILABLE) || defined(VOLTA_MMA_AVAILABLE)
|
||||
#endif // defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)
|
||||
}
|
||||
|
||||
template <int I_, int J_, typename T, data_layout ds_=DATA_LAYOUT_I_MAJOR>
|
||||
@@ -201,7 +201,7 @@ namespace ggml_cuda_mma {
|
||||
|
||||
static __device__ __forceinline__ int get_j(const int l) {
|
||||
if constexpr (I == 16 && J == 16) {
|
||||
#if defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#if defined(RDNA3)
|
||||
if constexpr (std::is_same_v<T, float> || std::is_same_v<T, int>) {
|
||||
// matrix C
|
||||
return 2 * l + (threadIdx.x / 16);
|
||||
@@ -210,11 +210,11 @@ namespace ggml_cuda_mma {
|
||||
return l;
|
||||
}
|
||||
#else
|
||||
// matrix C is the transposed matrix A&B on WMMA128
|
||||
// matrix C is the transposed matrix A&B on RDNA4
|
||||
return ne * (threadIdx.x / 16) + l;
|
||||
#endif // defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#endif // defined(RDNA3)
|
||||
} else if constexpr (I == 16 && J == 8) {
|
||||
// mmq input for WMMA128
|
||||
// mmq input for RDNA4
|
||||
return ne * (threadIdx.x / 16) + l;
|
||||
} else if constexpr (I == 16 && J == 4) {
|
||||
return ne * (threadIdx.x / 16) + l;
|
||||
@@ -573,7 +573,7 @@ namespace ggml_cuda_mma {
|
||||
static constexpr int I = I_;
|
||||
static constexpr int J = J_;
|
||||
static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;
|
||||
#if defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#if defined(RDNA3)
|
||||
static constexpr int ne = tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::ne;
|
||||
|
||||
half2 x[ne] = {{0.0f, 0.0f}};
|
||||
@@ -616,7 +616,7 @@ namespace ggml_cuda_mma {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
#endif // defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#endif // defined(RDNA3)
|
||||
};
|
||||
|
||||
template <int I_, int J_>
|
||||
@@ -694,7 +694,7 @@ namespace ggml_cuda_mma {
|
||||
};
|
||||
|
||||
static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) {
|
||||
#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
|
||||
tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret;
|
||||
#pragma unroll
|
||||
for (int l0 = 0; l0 < t.ne/2; ++l0) {
|
||||
@@ -705,7 +705,7 @@ namespace ggml_cuda_mma {
|
||||
#else
|
||||
NO_DEVICE_CODE;
|
||||
GGML_UNUSED(t);
|
||||
#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
|
||||
}
|
||||
|
||||
#if defined(TURING_MMA_AVAILABLE)
|
||||
@@ -726,7 +726,7 @@ namespace ggml_cuda_mma {
|
||||
|
||||
return ret;
|
||||
}
|
||||
#elif defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
|
||||
static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2(
|
||||
const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) {
|
||||
tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret;
|
||||
@@ -1171,7 +1171,7 @@ namespace ggml_cuda_mma {
|
||||
: "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));
|
||||
#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
|
||||
#elif defined(AMD_WMMA_AVAILABLE)
|
||||
#if defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#if defined(RDNA4)
|
||||
using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;
|
||||
halfx8_t& acc_frag = reinterpret_cast<halfx8_t&>(D.x[0]);
|
||||
const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);
|
||||
@@ -1180,7 +1180,7 @@ namespace ggml_cuda_mma {
|
||||
#else
|
||||
GGML_UNUSED_VARS(D, A, B);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#endif // defined(RDNA4)
|
||||
#elif defined(AMD_MFMA_AVAILABLE)
|
||||
// MFMA: FP16 input, FP32 accumulate, convert back to half2.
|
||||
using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;
|
||||
@@ -1218,7 +1218,7 @@ namespace ggml_cuda_mma {
|
||||
static __device__ __forceinline__ void mma(
|
||||
tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A,
|
||||
const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) {
|
||||
#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))
|
||||
tile<16, 8, half2> * D16 = (tile<16, 8, half2> *) &D;
|
||||
const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A;
|
||||
mma(D16[0], A16[0], B);
|
||||
@@ -1226,7 +1226,7 @@ namespace ggml_cuda_mma {
|
||||
#else
|
||||
GGML_UNUSED_VARS(D, A, B);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)
|
||||
}
|
||||
|
||||
template <data_layout dl_ab, data_layout dl_d>
|
||||
@@ -1373,14 +1373,14 @@ namespace ggml_cuda_mma {
|
||||
: "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));
|
||||
#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
|
||||
#elif defined(AMD_WMMA_AVAILABLE)
|
||||
#if defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#if defined(RDNA4)
|
||||
using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;
|
||||
using floatx8_t = __attribute__((ext_vector_type(8))) float;
|
||||
floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
|
||||
const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);
|
||||
const halfx8_t& b_frag = reinterpret_cast<const halfx8_t&>(B.x[0]);
|
||||
acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);
|
||||
#elif defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#elif defined(RDNA3)
|
||||
using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;
|
||||
using floatx8_t = __attribute__((ext_vector_type(8))) float;
|
||||
floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
|
||||
@@ -1390,7 +1390,7 @@ namespace ggml_cuda_mma {
|
||||
#else
|
||||
GGML_UNUSED_VARS(D, A, B);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // AMD_WMMA_128B_AVAILABLE
|
||||
#endif // RDNA4
|
||||
#elif defined(AMD_MFMA_AVAILABLE)
|
||||
using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;
|
||||
using floatx4_t = __attribute__((ext_vector_type(4))) float;
|
||||
@@ -1408,14 +1408,14 @@ namespace ggml_cuda_mma {
|
||||
static __device__ __forceinline__ void mma(
|
||||
tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) {
|
||||
#if defined(AMD_WMMA_AVAILABLE)
|
||||
#if defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#if defined(RDNA4)
|
||||
using bf16x8_t = __attribute__((ext_vector_type(8))) short;
|
||||
using floatx8_t = __attribute__((ext_vector_type(8))) float;
|
||||
floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
|
||||
const bf16x8_t& a_frag = reinterpret_cast<const bf16x8_t&>(A.x[0]);
|
||||
const bf16x8_t& b_frag = reinterpret_cast<const bf16x8_t&>(B.x[0]);
|
||||
acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag);
|
||||
#elif defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#elif defined(RDNA3)
|
||||
using bf16x16_t = __attribute__((ext_vector_type(16))) short;
|
||||
using floatx8_t = __attribute__((ext_vector_type(8))) float;
|
||||
floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);
|
||||
@@ -1425,7 +1425,7 @@ namespace ggml_cuda_mma {
|
||||
#else
|
||||
GGML_UNUSED_VARS(D, A, B);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#endif // defined(RDNA4)
|
||||
#elif defined(AMD_MFMA_AVAILABLE)
|
||||
using floatx4_t = __attribute__((ext_vector_type(4))) float;
|
||||
floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);
|
||||
@@ -1467,19 +1467,19 @@ namespace ggml_cuda_mma {
|
||||
#elif defined(AMD_WMMA_AVAILABLE)
|
||||
using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;
|
||||
int32x8_t * acc = (int32x8_t *) D.x;
|
||||
#if defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#if defined(RDNA4)
|
||||
using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;
|
||||
int32x2_t * a_vec = (int32x2_t *) A.x;
|
||||
int32x2_t * b_vec = (int32x2_t *) B.x;
|
||||
acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], true);
|
||||
acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[1], true, b_vec[1], acc[0], true);
|
||||
#elif defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#elif defined(RDNA3)
|
||||
using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
|
||||
int32x4_t * a_vec = (int32x4_t *) A.x;
|
||||
int32x4_t * b_vec = (int32x4_t *) B.x;
|
||||
acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], true);
|
||||
acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[1], true, b_vec[1], acc[0], true);
|
||||
#endif // AMD_WMMA_128B_AVAILABLE
|
||||
#endif // RDNA4
|
||||
#else
|
||||
GGML_UNUSED_VARS(D, A, B);
|
||||
NO_DEVICE_CODE;
|
||||
@@ -1556,7 +1556,7 @@ namespace ggml_cuda_mma {
|
||||
static __device__ __forceinline__ void mma(
|
||||
tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A,
|
||||
const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {
|
||||
#if defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)
|
||||
using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;
|
||||
halfx16_t * xD = (halfx16_t *) D.x;
|
||||
const halfx16_t * xA = (const halfx16_t *) A.x;
|
||||
@@ -1585,17 +1585,17 @@ namespace ggml_cuda_mma {
|
||||
#elif defined(AMD_WMMA_AVAILABLE)
|
||||
using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;
|
||||
int32x8_t * acc = (int32x8_t *) D.x;
|
||||
#if defined(AMD_WMMA_128B_AVAILABLE)
|
||||
#if defined(RDNA4)
|
||||
using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;
|
||||
int32x2_t * a_vec = (int32x2_t *) A.x;
|
||||
int32x2_t * b_vec = (int32x2_t *) B.x;
|
||||
acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], false);
|
||||
#elif defined(AMD_WMMA_256B_AVAILABLE)
|
||||
#elif defined(RDNA3)
|
||||
using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;
|
||||
int32x4_t * a_vec = (int32x4_t *) A.x;
|
||||
int32x4_t * b_vec = (int32x4_t *) B.x;
|
||||
acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], false);
|
||||
#endif // AMD_WMMA_128B_AVAILABLE
|
||||
#endif // RDNA4
|
||||
#else
|
||||
GGML_UNUSED(D);
|
||||
GGML_UNUSED(A);
|
||||
|
||||
@@ -170,13 +170,10 @@ void ggml_cuda_mul_mat_q(
|
||||
|
||||
// Fused gate/up/GLU: src0 is the up weight, dst the GLU output.
|
||||
const ggml_tensor * gate = fusion ? fusion->gate : nullptr;
|
||||
ggml_cuda_mm_fusion_args_device fusion_device = {};
|
||||
if (fusion) {
|
||||
GGML_ASSERT(gate && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale);
|
||||
// TODO MoE support
|
||||
GGML_ASSERT(gate && !ids && !fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale);
|
||||
GGML_ASSERT(gate->type == src0->type && ggml_are_same_shape(gate, src0) && ggml_are_same_stride(gate, src0));
|
||||
fusion_device.gate = gate->data;
|
||||
fusion_device.glu_op = fusion->glu_op;
|
||||
fusion_device.glu_limit = fusion->glu_limit;
|
||||
}
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS;
|
||||
@@ -242,7 +239,7 @@ void ggml_cuda_mul_mat_q(
|
||||
|
||||
// Each expert only sees ne12*n_expert_used/ne02 tokens on average.
|
||||
// On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {
|
||||
ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02;
|
||||
}
|
||||
}
|
||||
@@ -289,6 +286,13 @@ void ggml_cuda_mul_mat_q(
|
||||
ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));
|
||||
const int64_t s13 = ne12*s12;
|
||||
|
||||
ggml_cuda_mm_fusion_args_device fusion_device = {};
|
||||
if (gate) {
|
||||
fusion_device.gate = gate->data;
|
||||
fusion_device.glu_op = fusion->glu_op;
|
||||
fusion_device.glu_limit = fusion->glu_limit;
|
||||
}
|
||||
|
||||
const mmq_args args = {
|
||||
src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d,
|
||||
src0->type == GGML_TYPE_NVFP4 && use_native_fp4 ? src1_scale.ptr : nullptr,
|
||||
@@ -374,7 +378,7 @@ void ggml_cuda_mul_mat_q(
|
||||
ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,
|
||||
ne02, ne02, s02, s12, s2,
|
||||
ne03, ne13, s03, s13, s3,
|
||||
ne12, J_best, fusion_device};
|
||||
ne12, J_best};
|
||||
|
||||
ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);
|
||||
}
|
||||
@@ -470,7 +474,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
|
||||
}
|
||||
|
||||
if (amd_wmma_available(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc)) {
|
||||
// High expert counts are almost always better on MMQ due to
|
||||
// the synchronization overhead in the cuBLAS/hipBLAS path:
|
||||
// https://github.com/ggml-org/llama.cpp/pull/18202
|
||||
@@ -487,7 +491,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
|
||||
return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);
|
||||
case GGML_TYPE_IQ2_XS:
|
||||
case GGML_TYPE_IQ2_S:
|
||||
return GGML_CUDA_CC_IS_RDNA3_5(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc) || ne11 <= 128;
|
||||
return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -240,9 +240,6 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty
|
||||
if (GGML_CUDA_CC_IS_RDNA4(cc)) {
|
||||
return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA4_M(cc)) {
|
||||
return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
|
||||
return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
|
||||
}
|
||||
@@ -275,8 +272,6 @@ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_t
|
||||
return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
|
||||
#elif defined(RDNA4)
|
||||
return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
|
||||
#elif defined(RDNA4_M)
|
||||
return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
|
||||
#elif defined(RDNA3_5)
|
||||
return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
|
||||
#elif defined(RDNA3)
|
||||
|
||||
@@ -844,7 +844,7 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const
|
||||
return ne11 <= MMVF_MAX_BATCH_SIZE;
|
||||
}
|
||||
if (fp16_mma_hardware_available(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc)) {
|
||||
return ne11 <= 3;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA4(cc)) {
|
||||
|
||||
@@ -108,7 +108,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
||||
return MMVQ_PARAMETERS_RDNA4;
|
||||
#elif defined(RDNA3_0)
|
||||
return MMVQ_PARAMETERS_RDNA3_0;
|
||||
#elif defined(RDNA2) || defined(RDNA3_5) || defined(RDNA4_M)
|
||||
#elif defined(RDNA2) || defined(RDNA3_5)
|
||||
return MMVQ_PARAMETERS_RDNA2;
|
||||
#elif defined(GCN) || defined(CDNA)
|
||||
return MMVQ_PARAMETERS_GCN;
|
||||
@@ -128,7 +128,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3_0(cc)) {
|
||||
return MMVQ_PARAMETERS_RDNA3_0;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) {
|
||||
return MMVQ_PARAMETERS_RDNA2;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {
|
||||
@@ -299,7 +299,7 @@ int get_mmvq_mmid_max_batch(ggml_type type, int cc) {
|
||||
if (GGML_CUDA_CC_IS_RDNA4(cc)) {
|
||||
return get_mmvq_mmid_max_batch_rdna4(type);
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4_M(cc)) {
|
||||
if (GGML_CUDA_CC_IS_RDNA3(cc)) {
|
||||
return get_mmvq_mmid_max_batch_rdna3(type);
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA1(cc) || GGML_CUDA_CC_IS_RDNA2(cc)) {
|
||||
@@ -432,7 +432,7 @@ template <ggml_type type>
|
||||
static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {
|
||||
#if defined(RDNA4)
|
||||
return get_mmvq_mmid_max_batch_rdna4(type);
|
||||
#elif defined(RDNA3) || defined(RDNA4_M)
|
||||
#elif defined(RDNA3)
|
||||
return get_mmvq_mmid_max_batch_rdna3(type);
|
||||
#elif defined(RDNA2) || defined(RDNA1)
|
||||
return get_mmvq_mmid_max_batch_rdna1_rdna2(type);
|
||||
|
||||
@@ -402,10 +402,6 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx,
|
||||
ggml_cuda_kernel_launch(topk_moe_cuda<288, has_bias>, launch_params,
|
||||
logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
|
||||
break;
|
||||
case 384: // DeepSeek V4.1
|
||||
ggml_cuda_kernel_launch(topk_moe_cuda<384, has_bias>, launch_params,
|
||||
logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
|
||||
break;
|
||||
case 512:
|
||||
ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params,
|
||||
logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
|
||||
@@ -476,7 +472,7 @@ bool ggml_cuda_should_use_topk_moe(const ggml_tensor * gating_op,
|
||||
// must match an instantiation of launch_topk_moe_cuda: a power of 2 up to 512,
|
||||
// or one of the non-power-of-2 expert counts of supported models
|
||||
const int n_expert = ids->nb[1] / ids->nb[0];
|
||||
if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 384 && n_expert != 576) {
|
||||
if (((n_expert & (n_expert - 1)) != 0 || n_expert > 512) && n_expert != 288 && n_expert != 576) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
Vendored
+4
-12
@@ -216,13 +216,9 @@
|
||||
#define RDNA4
|
||||
#endif // defined(__GFX12__)
|
||||
|
||||
#if defined(__gfx1170__) || defined(__gfx1171__)
|
||||
#define RDNA4_M
|
||||
#endif // defined(__gfx1170__) || defined(__gfx1171__)
|
||||
|
||||
#if defined(__GFX11__) && !defined(RDNA4_M)
|
||||
#if defined(__GFX11__)
|
||||
#define RDNA3
|
||||
#endif // defined(__GFX11__) && !defined(RDNA4_M)
|
||||
#endif // defined(__GFX11__)
|
||||
|
||||
#if defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)
|
||||
#define RDNA3_5
|
||||
@@ -241,9 +237,9 @@
|
||||
#define RDNA1
|
||||
#endif // defined(__gfx1010__) || defined(__gfx1012__)
|
||||
|
||||
#if defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
|
||||
#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
|
||||
#define RDNA // For the entire family
|
||||
#endif // defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
|
||||
#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
|
||||
|
||||
#ifndef __has_builtin
|
||||
#define __has_builtin(x) 0
|
||||
@@ -258,10 +254,6 @@ typedef __hip_fp8_e4m3 __nv_fp8_e4m3;
|
||||
#define FP8_AVAILABLE
|
||||
#endif // HIP_VERSION >= 60300000
|
||||
|
||||
#if HIP_VERSION < 70000000
|
||||
#define __ballot_sync(mask, var) __ballot(var)
|
||||
#endif //HIP_VERSION < 70000000
|
||||
|
||||
typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));
|
||||
typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));
|
||||
static __device__ __forceinline__ int __vsubss4(const int a, const int b) {
|
||||
|
||||
@@ -373,9 +373,6 @@ static const struct ggml_backend_buffer_i ggml_backend_et_buffer_i = {
|
||||
/* .cpy_tensor = */ ggml_backend_et_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_et_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
static const char * ggml_backend_et_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
|
||||
|
||||
@@ -843,9 +843,6 @@ static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = {
|
||||
/* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_hexagon_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
// ** backend buffer type
|
||||
@@ -880,9 +877,6 @@ static ggml_backend_buffer_i ggml_backend_hexagon_host_buffer_interface = {
|
||||
/* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_hexagon_buffer_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ NULL,
|
||||
/* .get_tensor_n = */ NULL,
|
||||
/* .cpy_tensor_n = */ NULL,
|
||||
};
|
||||
|
||||
// ** backend buffer type
|
||||
|
||||
@@ -57,10 +57,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) {
|
||||
case GGML_OP_PAD: return HTP_OP_PAD;
|
||||
case GGML_OP_IM2COL: return HTP_OP_IM2COL;
|
||||
case GGML_OP_IM2COL_3D: return HTP_OP_IM2COL_3D;
|
||||
case GGML_OP_CONV_2D: return HTP_OP_CONV_2D;
|
||||
case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW;
|
||||
case GGML_OP_ROLL: return HTP_OP_ROLL;
|
||||
case GGML_OP_UPSCALE: return HTP_OP_UPSCALE;
|
||||
|
||||
case GGML_OP_UNARY:
|
||||
switch (ggml_get_unary_op(t)) {
|
||||
@@ -1793,6 +1790,11 @@ static bool htp_kparams_solve_softmax(
|
||||
return false;
|
||||
}
|
||||
|
||||
const int64_t ne0 = src0->ne[0];
|
||||
if ((ne0 > 32 && (ne0 & (32 - 1)) != 0) || ne0 > 131072) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
|
||||
const size_t dst_data_row_size = op->ne[0] * ggml_type_size(op->type);
|
||||
const uint32_t min_rows_per_thread = (uint32_t) (std::max<size_t>(1, (128 + dst_data_row_size - 1) / dst_data_row_size));
|
||||
@@ -2186,8 +2188,7 @@ static bool htp_kparams_solve_ssm_conv(
|
||||
target_max_tile = (std::max)(target_max_tile, 32u);
|
||||
target_max_tile = (std::min)(target_max_tile, 128u);
|
||||
|
||||
const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float);
|
||||
uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (raw_stride_bytes + n_t * sizeof(float))));
|
||||
uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (ncs + n_t) * sizeof(float)));
|
||||
d_inner_tile = (d_inner_tile / 32) * 32;
|
||||
if (d_inner_tile == 0) {
|
||||
d_inner_tile = 32;
|
||||
@@ -2374,183 +2375,6 @@ static bool htp_kparams_solve_pool_2d(
|
||||
return true;
|
||||
}
|
||||
|
||||
bool htp_kparams_solve_upscale(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * src0,
|
||||
const struct ggml_tensor * dst,
|
||||
struct htp_upscale_kernel_params * kparams
|
||||
) {
|
||||
memset(kparams, 0, sizeof(*kparams));
|
||||
|
||||
if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t src_x = (uint32_t) src0->ne[0];
|
||||
const uint32_t src_y = (uint32_t) src0->ne[1];
|
||||
const uint32_t src_z = (uint32_t) src0->ne[2];
|
||||
const uint32_t src_w = (uint32_t) src0->ne[3];
|
||||
|
||||
const uint32_t dst_x = (uint32_t) dst->ne[0];
|
||||
const uint32_t dst_y = (uint32_t) dst->ne[1];
|
||||
const uint32_t dst_z = (uint32_t) dst->ne[2];
|
||||
const uint32_t dst_w = (uint32_t) dst->ne[3];
|
||||
|
||||
if (src_x == 0 || src_y == 0 || src_z == 0 || src_w == 0 ||
|
||||
dst_x == 0 || dst_y == 0 || dst_z == 0 || dst_w == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool is_src_contig_row = (src0->nb[0] == sizeof(float));
|
||||
const bool is_src_strided_2d = (src0->nb[0] > sizeof(float)) &&
|
||||
(src0->nb[0] % sizeof(float) == 0) &&
|
||||
(src0->nb[1] == (size_t) src_x * src0->nb[0]) &&
|
||||
((size_t) src_x * src_y <= 0xFFFFu);
|
||||
|
||||
if ((!is_src_contig_row && !is_src_strided_2d) || dst->nb[0] != sizeof(float)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int32_t mode_flags = dst->op_params[0];
|
||||
const uint32_t mode_raw = (uint32_t) (mode_flags & 0xFF);
|
||||
if (mode_raw >= GGML_SCALE_MODE_COUNT) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool align_corners = (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) != 0;
|
||||
const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS) != 0;
|
||||
|
||||
float sf0 = (float) dst_x / (float) src_x;
|
||||
float sf1 = (float) dst_y / (float) src_y;
|
||||
float sf2 = (float) dst_z / (float) src_z;
|
||||
float sf3 = (float) dst_w / (float) src_w;
|
||||
float pixel_offset = 0.5f;
|
||||
|
||||
if (align_corners) {
|
||||
pixel_offset = 0.0f;
|
||||
sf0 = (dst_x > 1 && src_x > 1) ? (float) (dst_x - 1) / (float) (src_x - 1) : sf0;
|
||||
sf1 = (dst_y > 1 && src_y > 1) ? (float) (dst_y - 1) / (float) (src_y - 1) : sf1;
|
||||
}
|
||||
|
||||
const float inv_sf0 = 1.0f / sf0;
|
||||
const float inv_sf1 = 1.0f / sf1;
|
||||
const float inv_sf2 = 1.0f / sf2;
|
||||
const float inv_sf3 = 1.0f / sf3;
|
||||
|
||||
const float filter_radius0 = antialias ? (std::max)(1.0f, inv_sf0) : 0.0f;
|
||||
const float filter_radius1 = antialias ? (std::max)(1.0f, inv_sf1) : 0.0f;
|
||||
const float invscale0 = antialias ? (1.0f / filter_radius0) : 0.0f;
|
||||
const float invscale1 = antialias ? (1.0f / filter_radius1) : 0.0f;
|
||||
|
||||
const uint32_t total_slices = dst_z * dst_w;
|
||||
uint32_t n_threads = htp_session_n_threads(sess);
|
||||
|
||||
struct htp_upscale_vtcm_layout layout;
|
||||
uint32_t dst_nrows_per_chunk = 0;
|
||||
while (n_threads > 0) {
|
||||
if (htp_upscale_solve_layout(&layout, src_x, src_y, dst_x, dst_y, inv_sf1, filter_radius1, n_threads,
|
||||
htp_session_vtcm_size(sess), &dst_nrows_per_chunk)) {
|
||||
break;
|
||||
}
|
||||
n_threads--;
|
||||
}
|
||||
if (n_threads == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t lut_x_size = 0;
|
||||
size_t elem_y_sz = sizeof(uint32_t);
|
||||
if (mode_raw == HTP_UPSCALE_MODE_BICUBIC) {
|
||||
lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_bicubic_x), 128);
|
||||
elem_y_sz = sizeof(struct upscale_bicubic_y);
|
||||
} else if (antialias) {
|
||||
lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_antialias_x), 128);
|
||||
elem_y_sz = sizeof(struct upscale_antialias_y);
|
||||
} else if (mode_raw == HTP_UPSCALE_MODE_BILINEAR) {
|
||||
const uint32_t n_chunks = (dst_x + 31) / 32;
|
||||
lut_x_size = hex_round_up(n_chunks * sizeof(struct upscale_bilinear_x_chunk), 128);
|
||||
elem_y_sz = sizeof(struct upscale_bilinear_y);
|
||||
} else {
|
||||
lut_x_size = hex_round_up(dst_x * sizeof(uint32_t), 128);
|
||||
}
|
||||
const size_t total_lut_size = lut_x_size + hex_round_up(dst_y * elem_y_sz, 128);
|
||||
if (total_lut_size > HTP_DDR_SPAD_SIZE) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t chunks_per_slice = (dst_y + dst_nrows_per_chunk - 1) / dst_nrows_per_chunk;
|
||||
const uint32_t total_chunks = total_slices * chunks_per_slice;
|
||||
|
||||
n_threads = (std::min)(n_threads, total_chunks > 0 ? total_chunks : 1);
|
||||
htp_upscale_vtcm_layout_build(&layout, src_x, src_y, dst_x, dst_y, dst_nrows_per_chunk, inv_sf1, filter_radius1, n_threads);
|
||||
|
||||
const uint32_t slice_dst_bytes = dst_x * dst_y * sizeof(float);
|
||||
const uint32_t chunk_dst_bytes = dst_x * dst_nrows_per_chunk * sizeof(float);
|
||||
const bool dst_contig = (dst->nb[1] == (size_t) dst_x * sizeof(float)) &&
|
||||
(dst_z <= 1 || dst->nb[2] == (size_t) dst_y * dst->nb[1]) &&
|
||||
(dst_w <= 1 || dst->nb[3] == (size_t) dst_z * dst->nb[2]);
|
||||
|
||||
uint32_t chunks_per_group = 0;
|
||||
if (dst_contig) {
|
||||
const uint32_t slices_per_group = (slice_dst_bytes > 0)
|
||||
? (HEX_L2_LINE_SIZE / hex_gcd_u32(slice_dst_bytes, HEX_L2_LINE_SIZE))
|
||||
: 1;
|
||||
if (slices_per_group == 1 && (chunk_dst_bytes % HEX_L2_LINE_SIZE == 0)) {
|
||||
chunks_per_group = 1;
|
||||
} else {
|
||||
chunks_per_group = slices_per_group * chunks_per_slice;
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t kernel_type = HTP_UPSCALE_KERNEL_BILINEAR;
|
||||
if (mode_raw == HTP_UPSCALE_MODE_NEAREST) {
|
||||
if (dst_x == 2 * src_x && dst_y == 2 * src_y && !align_corners) {
|
||||
kernel_type = HTP_UPSCALE_KERNEL_NEAREST_2X;
|
||||
} else if (dst_x == 4 * src_x && dst_y == 4 * src_y && !align_corners) {
|
||||
kernel_type = HTP_UPSCALE_KERNEL_NEAREST_4X;
|
||||
} else {
|
||||
kernel_type = HTP_UPSCALE_KERNEL_NEAREST;
|
||||
}
|
||||
} else if (mode_raw == HTP_UPSCALE_MODE_BICUBIC) {
|
||||
kernel_type = HTP_UPSCALE_KERNEL_BICUBIC;
|
||||
} else if (antialias) {
|
||||
kernel_type = HTP_UPSCALE_KERNEL_BILINEAR_AA;
|
||||
} else if (dst_x == 2 * src_x && dst_y == 2 * src_y && !align_corners && (src_x % 32 == 0)) {
|
||||
kernel_type = HTP_UPSCALE_KERNEL_BILINEAR_2X;
|
||||
} else {
|
||||
kernel_type = HTP_UPSCALE_KERNEL_BILINEAR;
|
||||
}
|
||||
|
||||
kparams->kernel_type = kernel_type;
|
||||
kparams->mode = (uint8_t) mode_raw;
|
||||
kparams->align_corners = align_corners ? 1 : 0;
|
||||
kparams->antialias = antialias ? 1 : 0;
|
||||
|
||||
kparams->inv_sf0 = inv_sf0;
|
||||
kparams->inv_sf1 = inv_sf1;
|
||||
kparams->inv_sf2 = inv_sf2;
|
||||
kparams->inv_sf3 = inv_sf3;
|
||||
kparams->pixel_offset = pixel_offset;
|
||||
|
||||
kparams->filter_radius0 = filter_radius0;
|
||||
kparams->filter_radius1 = filter_radius1;
|
||||
kparams->invscale0 = invscale0;
|
||||
kparams->invscale1 = invscale1;
|
||||
|
||||
kparams->n_threads = n_threads;
|
||||
kparams->chunks_per_group = chunks_per_group;
|
||||
|
||||
kparams->dst_nrows_per_chunk = dst_nrows_per_chunk;
|
||||
kparams->chunks_per_slice = chunks_per_slice;
|
||||
kparams->total_chunks = total_chunks;
|
||||
kparams->vtcm_size = (uint32_t) layout.total_bytes;
|
||||
|
||||
kparams->div_chunks_per_slice = init_fastdiv_values(chunks_per_slice > 0 ? chunks_per_slice : 1);
|
||||
kparams->div_dst_z = init_fastdiv_values(dst_z > 0 ? dst_z : 1);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool htp_kparams_solve_concat(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * op,
|
||||
@@ -2905,8 +2729,7 @@ static bool htp_kparams_solve_im2col(
|
||||
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
|
||||
|
||||
struct htp_im2col_vtcm_layout layout;
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads,
|
||||
htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_PATCHEMBED_DMA, KH, patch_stride));
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
|
||||
|
||||
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
|
||||
kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
|
||||
@@ -2955,24 +2778,18 @@ static bool htp_kparams_solve_im2col(
|
||||
const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
|
||||
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
|
||||
|
||||
// threads work on output-col blocks, so a single output row (1D) can use them all
|
||||
const uint32_t nbpr = (OW + owb - 1) / owb;
|
||||
const uint64_t nblocks = (uint64_t) total_rows * nbpr;
|
||||
const uint32_t blk_threads = (uint32_t) (nblocks < max_threads ? nblocks : max_threads);
|
||||
|
||||
struct htp_im2col_vtcm_layout layout;
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, blk_threads,
|
||||
htp_im2col_scratch_bytes(HTP_IM2COL_KERNEL_BLOCKED_DMA, KH, patch_stride));
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
|
||||
|
||||
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
|
||||
kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
|
||||
kparams->n_threads = (uint8_t) blk_threads;
|
||||
kparams->n_threads = (uint8_t) n_threads;
|
||||
kparams->vtcm_size = (uint32_t) layout.total_bytes;
|
||||
kparams->src_row_bytes = src_row_bytes;
|
||||
kparams->dst_row_bytes = dst_row_bytes;
|
||||
kparams->owb = owb;
|
||||
kparams->wb = (uint32_t) Wb;
|
||||
kparams->nbpr = nbpr;
|
||||
kparams->nbpr = (OW + owb - 1) / owb;
|
||||
kparams->N = N;
|
||||
kparams->IC = IC;
|
||||
kparams->ID = 1;
|
||||
@@ -2989,7 +2806,6 @@ static bool htp_kparams_solve_im2col(
|
||||
kparams->rows_per_chunk = rows_per_chunk;
|
||||
kparams->div_OH = init_fastdiv_values(OH);
|
||||
kparams->div_OD = init_fastdiv_values(1);
|
||||
kparams->div_nbpr = init_fastdiv_values(nbpr);
|
||||
return true;
|
||||
}
|
||||
if (owb == 1) break;
|
||||
@@ -3091,7 +2907,7 @@ static bool htp_kparams_solve_im2col_3d(
|
||||
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
|
||||
|
||||
struct htp_im2col_vtcm_layout layout;
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256);
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
|
||||
|
||||
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
|
||||
kparams->kernel_type = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
|
||||
@@ -3141,7 +2957,7 @@ static bool htp_kparams_solve_im2col_3d(
|
||||
const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
|
||||
|
||||
struct htp_im2col_vtcm_layout layout;
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads, 256);
|
||||
htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
|
||||
|
||||
if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
|
||||
kparams->kernel_type = HTP_IM2COL_KERNEL_BLOCKED_DMA;
|
||||
@@ -3470,545 +3286,6 @@ static bool htp_kparams_solve_solve_tri(
|
||||
return true;
|
||||
}
|
||||
|
||||
// Implicit GEMM over sliding windows of fp16 input croutons (HMX activation:single), no im2col
|
||||
static bool htp_kparams_conv2d_win(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * dst,
|
||||
struct htp_conv2d_kernel_params * kp
|
||||
) {
|
||||
if (htp_session_n_hmx(sess) == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const struct ggml_tensor * src0 = dst->src[0];
|
||||
const struct ggml_tensor * src1 = dst->src[1];
|
||||
const int32_t s0 = ggml_get_op_params_i32(dst, 0);
|
||||
const int32_t s1 = ggml_get_op_params_i32(dst, 1);
|
||||
const int32_t p0 = ggml_get_op_params_i32(dst, 2);
|
||||
const int32_t p1 = ggml_get_op_params_i32(dst, 3);
|
||||
const int32_t d0 = ggml_get_op_params_i32(dst, 4);
|
||||
const int32_t d1 = ggml_get_op_params_i32(dst, 5);
|
||||
|
||||
const uint32_t KW = src0->ne[0];
|
||||
const uint32_t KH = src0->ne[1];
|
||||
const uint32_t Cin = src0->ne[2];
|
||||
const uint32_t Cout = src0->ne[3];
|
||||
const uint32_t IW = src1->ne[0];
|
||||
const uint32_t IH = src1->ne[1];
|
||||
const uint32_t OW = dst->ne[0];
|
||||
const uint32_t OH = dst->ne[1];
|
||||
const uint32_t N = dst->ne[3];
|
||||
const size_t es = ggml_type_size(src0->type);
|
||||
|
||||
if (s0 != 1 || s1 != 1 || p0 < 0 || p1 < 0 || dst->nb[0] != sizeof(float)) {
|
||||
return false;
|
||||
}
|
||||
if (dst->nb[1] < OW * sizeof(float) || (Cout > 1 && dst->nb[2] < OW * sizeof(float))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool whcn = src1->nb[0] == sizeof(float) && src1->nb[1] >= IW * sizeof(float) && (Cin == 1 || src1->nb[2] >= IH * src1->nb[1]);
|
||||
const bool cwhn = !whcn && src1->nb[2] == sizeof(float) && src1->nb[0] == Cin * sizeof(float) && src1->nb[1] >= IW * src1->nb[0];
|
||||
if (!whcn && !cwhn) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool w_rows = src0->nb[0] == es && (KH == 1 || src0->nb[1] == KW * es) && (Cin == 1 || src0->nb[2] == KW * KH * es);
|
||||
const bool co_c = (Cout == 1 || src0->nb[3] == es) && (Cin == 1 || src0->nb[2] >= (std::min)(Cout, 32u) * es);
|
||||
if (!w_rows && !co_c) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// short output rows: fold the kw taps into channels so the virtual row is OW wide and output rows are contiguous
|
||||
const bool fold = (whcn || (cwhn && d0 == 1)) && KW > 1 && KW * Cin <= 32 && OW < 64 && KW * KH <= 16 && Cout >= 16;
|
||||
|
||||
const uint32_t T = KW * KH;
|
||||
const uint32_t n_ch = fold ? KW * Cin : Cin;
|
||||
const uint32_t n_cc = (n_ch + 31) / 32;
|
||||
const uint32_t rem = n_ch - (n_cc - 1) * 32;
|
||||
const uint32_t stop = (rem + 7) / 8 * 8 - 1;
|
||||
const uint32_t SR = fold ? OW : IW + 2 * p0;
|
||||
const uint32_t IHS = IH + 2 * p1;
|
||||
const uint32_t n_wt = fold ? KH : T;
|
||||
const uint32_t moff = (KH - 1) * d1 * SR + (fold ? 0 : (KW - 1) * d0);
|
||||
const uint32_t halo = moff / 32 + 1;
|
||||
const uint32_t n_rt = (Cout + 31) / 32;
|
||||
const uint64_t n_q = (uint64_t) (N * IHS - (IHS - OH) - 1) * SR + OW;
|
||||
const uint32_t nth = kp->n_threads;
|
||||
const size_t tile = 2048;
|
||||
|
||||
if ((uint64_t) N * IHS * SR + moff + 64 >= (1ull << 31)) {
|
||||
return false;
|
||||
}
|
||||
const uint32_t n_tiles = (uint32_t) ((n_q + 31) / 32);
|
||||
|
||||
// weight packing scratch
|
||||
uint32_t w_cs = 0;
|
||||
size_t wraw = 0;
|
||||
size_t wscr = 0;
|
||||
if (w_rows) {
|
||||
const uint32_t kpw = es == 2 ? 64 : 32;
|
||||
for (uint32_t cs = 32; cs >= 2; cs /= 2) {
|
||||
w_cs = cs;
|
||||
wraw = (size_t) 32 * ((cs * T + kpw - 1) / kpw) * 128;
|
||||
if (3 * wraw + 64 * 128 <= 256 * 1024 || (fold && cs == 32)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// enough packing units for all threads
|
||||
const uint32_t kpw2 = es == 2 ? 64 : 32;
|
||||
while (!fold && w_cs > 2 && n_rt * ((Cin + 31) / 32) * (32 / w_cs) < kp->n_threads) {
|
||||
w_cs /= 2;
|
||||
}
|
||||
wraw = (std::min)(wraw, (size_t) 32 * ((w_cs * T + kpw2 - 1) / kpw2) * 128);
|
||||
wscr = 3 * wraw + 64 * 128 + 128;
|
||||
} else {
|
||||
w_cs = fold ? KW : (std::min)(T, 16u);
|
||||
while (!fold && w_cs > 1 && n_rt * ((Cin + 31) / 32) * ((T + w_cs - 1) / w_cs) < kp->n_threads) {
|
||||
w_cs = (w_cs + 1) / 2;
|
||||
}
|
||||
wscr = 2 * (size_t) w_cs * 32 * 128;
|
||||
}
|
||||
|
||||
uint32_t rt_block = n_rt;
|
||||
uint32_t ct = (std::max)(4 * halo, (n_tiles + 3) / 4);
|
||||
ct = (std::min)((std::max)(ct, 8u), 512u);
|
||||
ct = (std::min)(ct, n_tiles);
|
||||
uint32_t seg_b = 8;
|
||||
// longer output rows make the output DMA more efficient
|
||||
uint32_t seg_t = (std::min)(32u, (std::max)(8u, (OW + 31) / 32));
|
||||
|
||||
while (rt_block > 1 && (size_t) rt_block * n_wt * n_cc * tile > htp_session_vtcm_size(sess) / 2) {
|
||||
rt_block = (rt_block + 1) / 2;
|
||||
}
|
||||
|
||||
size_t stage = 0, thr = 0, act = 0, out = 0, w_size = 0, total = 0;
|
||||
uint32_t n_bc = 0;
|
||||
while (true) {
|
||||
n_bc = ct + halo;
|
||||
stage = cwhn ? (size_t) seg_b * 32 * Cin * sizeof(float) + 256 : (size_t) n_ch * seg_b * 32 * sizeof(float);
|
||||
if (fold) {
|
||||
// raw input rows of the staged pixels with 128 bytes of read slack on both sides
|
||||
stage = (size_t) Cin * ((seg_b * 32 + OW - 1) / OW + 1) * IW * sizeof(float) + 256;
|
||||
}
|
||||
stage = hex_round_up(stage, 128);
|
||||
thr = hex_round_up((std::max)(2 * stage + (size_t) HTP_CONV2D_WIN_OBUF * 32 * seg_t * 32 * sizeof(float), wscr), tile);
|
||||
w_size = (size_t) rt_block * n_wt * n_cc * tile;
|
||||
act = (size_t) n_cc * n_bc * tile;
|
||||
out = (size_t) rt_block * ct * tile;
|
||||
struct htp_conv2d_vtcm_layout L;
|
||||
htp_conv2d_vtcm_layout_build(&L, rt_block, w_size, act, out, thr, nth);
|
||||
total = L.total_bytes;
|
||||
if (total <= htp_session_vtcm_size(sess)) {
|
||||
break;
|
||||
}
|
||||
if (ct > 64) {
|
||||
ct /= 2;
|
||||
} else if (rt_block > 1) {
|
||||
rt_block = (rt_block + 1) / 2;
|
||||
} else if (seg_b > 1 || seg_t > 1) {
|
||||
seg_b = (std::max)(seg_b / 2, 1u);
|
||||
seg_t = (std::max)(seg_t / 2, 1u);
|
||||
} else if (ct > 1) {
|
||||
ct /= 2;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (hex_round_up((size_t) T * sizeof(uint32_t), 128) + 2 * hex_round_up((size_t) ct, 128) > HTP_DDR_SPAD_SIZE) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t TG = w_cs;
|
||||
const uint32_t CS = w_cs;
|
||||
const uint32_t n_tg = (T + TG - 1) / TG;
|
||||
const uint32_t n_cs = (32 + CS - 1) / CS;
|
||||
|
||||
kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_WINDOW;
|
||||
kp->n_threads = (uint8_t) nth;
|
||||
kp->cwhn = cwhn ? 1 : 0;
|
||||
kp->w_kmajor = !w_rows ? 1 : 0;
|
||||
kp->n_row_tiles = n_rt;
|
||||
kp->ct_chunk = ct;
|
||||
kp->rt_block = rt_block;
|
||||
kp->stage_size = stage;
|
||||
kp->thr_size = thr;
|
||||
kp->vtcm_size = total;
|
||||
kp->window.fold = fold;
|
||||
kp->window.n_cc = n_cc;
|
||||
kp->window.act_rt_last = 0x780 | 0x2 | (stop << 2);
|
||||
kp->window.sr = SR;
|
||||
kp->window.ihs = IHS;
|
||||
kp->window.max_off = moff;
|
||||
kp->window.n_bc = n_bc;
|
||||
kp->window.seg_b = seg_b;
|
||||
kp->window.seg_t = seg_t;
|
||||
kp->window.w_cs = w_cs;
|
||||
kp->window.div_sr = init_fastdiv_values(SR);
|
||||
kp->window.div_ihs = init_fastdiv_values(IHS);
|
||||
kp->window.div_kw = init_fastdiv_values(KW);
|
||||
kp->window.div_tg = init_fastdiv_values(n_tg);
|
||||
kp->window.div_cc = init_fastdiv_values(n_cc);
|
||||
kp->window.div_cs = init_fastdiv_values(n_cs);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Tiled implicit GEMM, MMA on HMX if available, else HVX
|
||||
static bool htp_kparams_conv2d_tiled(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * dst,
|
||||
struct htp_conv2d_kernel_params * kp
|
||||
) {
|
||||
if (htp_session_n_hmx(sess) == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const struct ggml_tensor * src0 = dst->src[0];
|
||||
const struct ggml_tensor * src1 = dst->src[1];
|
||||
const int32_t s0 = ggml_get_op_params_i32(dst, 0);
|
||||
const int32_t d0 = ggml_get_op_params_i32(dst, 4);
|
||||
|
||||
const uint32_t KW = src0->ne[0];
|
||||
const uint32_t KH = src0->ne[1];
|
||||
const uint32_t Cin = src0->ne[2];
|
||||
const uint32_t Cout = src0->ne[3];
|
||||
const uint32_t OW = dst->ne[0];
|
||||
const uint32_t OH = dst->ne[1];
|
||||
const uint32_t N = dst->ne[3];
|
||||
const size_t es = ggml_type_size(src0->type);
|
||||
|
||||
if (s0 != 1 || dst->nb[0] != sizeof(float)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// kernel k order must match the staged input: (cin, kh, kw) for WHCN input, (kh, kw, cin) for CWHN input.
|
||||
// Kernel is either k-contiguous rows per cout (WHCN only) or cout-contiguous with a uniform k stride.
|
||||
const bool whcn = src1->nb[0] == sizeof(float);
|
||||
const bool cwhn = !whcn && src1->nb[2] == sizeof(float) && src1->nb[0] == Cin * sizeof(float);
|
||||
const bool co_c = Cout == 1 || src0->nb[3] == es;
|
||||
const bool w_rows = whcn && src0->nb[0] == es && (KH == 1 || src0->nb[1] == KW * es) && (Cin == 1 || src0->nb[2] == KW * KH * es);
|
||||
const size_t ks = whcn ? src0->nb[0] : src0->nb[2];
|
||||
bool kmajor = false;
|
||||
if (whcn) {
|
||||
kmajor = co_c && ks >= Cout * es && (KH == 1 || src0->nb[1] == KW * ks) && (Cin == 1 || src0->nb[2] == KW * KH * ks);
|
||||
} else if (cwhn) {
|
||||
kmajor = co_c && ks >= Cout * es && src0->nb[0] == Cin * ks && (KH == 1 || src0->nb[1] == KW * Cin * ks);
|
||||
}
|
||||
if (!w_rows && !kmajor) {
|
||||
return false;
|
||||
}
|
||||
kmajor = kmajor && !w_rows;
|
||||
|
||||
const uint32_t K = KW * KH * Cin;
|
||||
const uint32_t kg = !cwhn ? K : d0 == 1 ? KW * Cin : Cin;
|
||||
const uint32_t n_grp = K / kg;
|
||||
const uint32_t kg_p = hex_round_up(kg, 32);
|
||||
const uint32_t k_pad = hex_round_up(n_grp * kg_p, 64);
|
||||
const uint32_t n_kt = k_pad / 32;
|
||||
const uint32_t n_rt = (Cout + 31) / 32;
|
||||
const uint32_t n_owt = (OW + 31) / 32;
|
||||
const uint32_t n_ct = N * OH * n_owt;
|
||||
const uint32_t wk = (std::min)(k_pad, 512u);
|
||||
const uint32_t nth = htp_session_n_threads(sess);
|
||||
const size_t tile = 2048;
|
||||
|
||||
uint32_t rt_block = n_rt;
|
||||
uint32_t ct = (std::min)(n_ct, 64u);
|
||||
uint32_t seg = (std::min)(8u, n_owt);
|
||||
uint32_t n_stage = 1;
|
||||
uint32_t sr = 0;
|
||||
size_t stage = 0, ostage = 0, prow = 0, wpack = 0;
|
||||
size_t w_size = 0, act_size = 0, out_size = 0, thr_size = 0, total = 0;
|
||||
|
||||
while (true) {
|
||||
const uint32_t wb = 32 * seg + (KW - 1) * d0;
|
||||
sr = cwhn ? hex_round_up(wb * Cin, 64) : hex_round_up(wb, 32);
|
||||
stage = cwhn ? (size_t) KH * sr * sizeof(float) : (size_t) Cin * KH * sr * sizeof(float);
|
||||
ostage = 2 * 32 * (size_t) seg * 32 * sizeof(float);
|
||||
prow = cwhn ? stage / 2 + 256 + 64 * 128 : 0;
|
||||
wpack = kmajor ? 2 * (size_t) wk * 64 * sizeof(float) + 96 * 128 : 2 * (size_t) 32 * wk * (sizeof(float) + sizeof(uint16_t));
|
||||
thr_size = hex_round_up((std::max)(stage + ostage + prow, wpack), tile);
|
||||
w_size = (size_t) rt_block * n_kt * tile;
|
||||
act_size = (size_t) ct * n_kt * tile;
|
||||
out_size = (size_t) rt_block * ct * tile;
|
||||
struct htp_conv2d_vtcm_layout L;
|
||||
htp_conv2d_vtcm_layout_build(&L, rt_block, w_size, act_size, out_size, thr_size, nth);
|
||||
total = L.total_bytes;
|
||||
if (total <= htp_session_vtcm_size(sess)) {
|
||||
break;
|
||||
}
|
||||
if (ct > 8) {
|
||||
ct /= 2;
|
||||
} else if (seg > 1) {
|
||||
seg--;
|
||||
} else if (rt_block > 1) {
|
||||
rt_block = (rt_block + 1) / 2;
|
||||
} else if (ct > 1) {
|
||||
ct /= 2;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Second stage buffer: DMA of the next segment overlaps HVX pack of this one.
|
||||
// Skip large stages. They already fill VTCM, and the extra DMA slows HMX.
|
||||
{
|
||||
const size_t thr2 = hex_round_up((std::max)(2 * stage + ostage + prow, wpack), tile);
|
||||
struct htp_conv2d_vtcm_layout L2;
|
||||
htp_conv2d_vtcm_layout_build(&L2, rt_block, w_size, act_size, out_size, thr2, nth);
|
||||
if (stage <= 128 * 1024 && L2.total_bytes <= htp_session_vtcm_size(sess)) {
|
||||
n_stage = 2;
|
||||
thr_size = thr2;
|
||||
total = L2.total_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
kp->kernel_type = (uint8_t) HTP_CONV2D_KERNEL_HMX_TILED;
|
||||
kp->n_threads = (uint8_t) nth;
|
||||
kp->cwhn = cwhn ? 1 : 0;
|
||||
kp->w_kmajor = kmajor ? 1 : 0;
|
||||
kp->n_row_tiles = n_rt;
|
||||
kp->ct_chunk = ct;
|
||||
kp->rt_block = rt_block;
|
||||
kp->stage_size = stage;
|
||||
kp->thr_size = thr_size;
|
||||
kp->vtcm_size = total;
|
||||
kp->tiled.w_kstride = ks;
|
||||
kp->tiled.k_size = K;
|
||||
kp->tiled.k_group = kg;
|
||||
kp->tiled.k_group_pad = kg_p;
|
||||
kp->tiled.n_groups = n_grp;
|
||||
kp->tiled.n_k_tiles = n_kt;
|
||||
kp->tiled.n_ow_tiles = n_owt;
|
||||
kp->tiled.seg_tiles = seg;
|
||||
kp->tiled.stage_row = sr;
|
||||
kp->tiled.n_stage = n_stage;
|
||||
kp->tiled.wk_chunk = wk;
|
||||
kp->tiled.div_ow_tiles = init_fastdiv_values(n_owt);
|
||||
kp->tiled.div_oh = init_fastdiv_values(OH);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool htp_kparams_solve_conv2d_unchecked(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * dst,
|
||||
struct htp_conv2d_kernel_params * kparams
|
||||
) {
|
||||
memset(kparams, 0, sizeof(*kparams));
|
||||
|
||||
const uint32_t total_rows = (uint32_t) (dst->ne[1] * dst->ne[2] * dst->ne[3]);
|
||||
kparams->total_rows = total_rows;
|
||||
kparams->n_threads = total_rows == 0 ? 1 : (std::min)((uint32_t) htp_session_n_threads(sess), total_rows);
|
||||
|
||||
if (htp_kparams_conv2d_win(sess, dst, kparams)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return htp_kparams_conv2d_tiled(sess, dst, kparams);
|
||||
}
|
||||
|
||||
static bool htp_kparams_solve_conv2d(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * op,
|
||||
struct htp_conv2d_kernel_params * kparams
|
||||
) {
|
||||
memset(kparams, 0, sizeof(*kparams));
|
||||
|
||||
const struct ggml_tensor * src0 = op->src[0];
|
||||
const struct ggml_tensor * src1 = op->src[1];
|
||||
const struct ggml_tensor * dst = op;
|
||||
|
||||
if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) ||
|
||||
src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (src0->ne[2] != src1->ne[2]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int32_t s0 = ggml_get_op_params_i32(op, 0);
|
||||
const int32_t s1 = ggml_get_op_params_i32(op, 1);
|
||||
const int32_t p0 = ggml_get_op_params_i32(op, 2);
|
||||
const int32_t p1 = ggml_get_op_params_i32(op, 3);
|
||||
const int32_t d0 = ggml_get_op_params_i32(op, 4);
|
||||
const int32_t d1 = ggml_get_op_params_i32(op, 5);
|
||||
|
||||
const int64_t ow = (src1->ne[0] + 2 * p0 - d0 * (src0->ne[0] - 1) - 1) / s0 + 1;
|
||||
const int64_t oh = (src1->ne[1] + 2 * p1 - d1 * (src0->ne[1] - 1) - 1) / s1 + 1;
|
||||
if (ow != dst->ne[0] || oh != dst->ne[1] || src0->ne[3] != dst->ne[2] || src1->ne[3] != dst->ne[3]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return htp_kparams_solve_conv2d_unchecked(sess, op, kparams);
|
||||
}
|
||||
|
||||
static bool htp_kparams_solve_conv2d_dw(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * op,
|
||||
struct htp_conv2d_dw_kernel_params * kparams
|
||||
) {
|
||||
memset(kparams, 0, sizeof(*kparams));
|
||||
|
||||
const struct ggml_tensor * src0 = op->src[0];
|
||||
const struct ggml_tensor * src1 = op->src[1];
|
||||
const struct ggml_tensor * dst = op;
|
||||
|
||||
if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) ||
|
||||
src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (src0->ne[2] != 1 || src0->ne[3] != src1->ne[2]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (src0->ne[0] * src0->ne[1] > 64) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int32_t s0 = ggml_get_op_params_i32(op, 0);
|
||||
const int32_t s1 = ggml_get_op_params_i32(op, 1);
|
||||
const int32_t p0 = ggml_get_op_params_i32(op, 2);
|
||||
const int32_t p1 = ggml_get_op_params_i32(op, 3);
|
||||
const int32_t d0 = ggml_get_op_params_i32(op, 4);
|
||||
const int32_t d1 = ggml_get_op_params_i32(op, 5);
|
||||
|
||||
if (s0 <= 0 || s1 <= 0 || d0 <= 0 || d1 <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int64_t ow = (src1->ne[0] + 2 * p0 - d0 * (src0->ne[0] - 1) - 1) / s0 + 1;
|
||||
const int64_t oh = (src1->ne[1] + 2 * p1 - d1 * (src0->ne[1] - 1) - 1) / s1 + 1;
|
||||
if (ow != dst->ne[0] || oh != dst->ne[1] || dst->ne[2] != src1->ne[2] || dst->ne[3] != src1->ne[3]) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool cwhn = ggml_is_contiguous_channels(src1);
|
||||
if (!cwhn && !ggml_is_contiguous(src1)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!cwhn && (d0 != 1 || (s0 != 1 && s0 != 2))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t c = (uint32_t) src1->ne[2];
|
||||
const uint32_t n = (uint32_t) src1->ne[3];
|
||||
const uint32_t w = (uint32_t) src1->ne[0];
|
||||
const uint32_t h = (uint32_t) src1->ne[1];
|
||||
const uint32_t kw = (uint32_t) src0->ne[0];
|
||||
const uint32_t kh = (uint32_t) src0->ne[1];
|
||||
const uint32_t ow_u = (uint32_t) ow;
|
||||
const uint32_t oh_u = (uint32_t) oh;
|
||||
|
||||
const uint32_t row_src_bytes = cwhn
|
||||
? (w * c * (uint32_t) sizeof(float))
|
||||
: (w * (uint32_t) sizeof(float));
|
||||
const uint32_t row_dst_bytes = cwhn
|
||||
? (ow_u * c * (uint32_t) sizeof(float))
|
||||
: (ow_u * (uint32_t) sizeof(float));
|
||||
|
||||
uint8_t kernel_type = HTP_CONV2D_DW_KERNEL_UNSUPPORTED;
|
||||
if (cwhn) {
|
||||
kernel_type = HTP_CONV2D_DW_KERNEL_CWHN;
|
||||
} else if ((w & 31) == 0 && kw == 3 && kh == 3 && p0 == 1 && p1 == 1 && d0 == 1 && d1 == 1 && s0 == 1 && s1 == 1) {
|
||||
kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S1_K3X3;
|
||||
} else if ((w & 31) == 0 && kw == 3 && kh == 3 && p0 == 1 && p1 == 1 && d0 == 1 && d1 == 1 && s0 == 2 && s1 == 2) {
|
||||
if (w == 32 && ow_u == 16) {
|
||||
kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W32;
|
||||
} else if (w == 64 && ow_u == 32) {
|
||||
kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W64;
|
||||
} else {
|
||||
kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3;
|
||||
}
|
||||
} else if (s0 == 1) {
|
||||
kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S1;
|
||||
} else {
|
||||
kernel_type = HTP_CONV2D_DW_KERNEL_WHCN_S2;
|
||||
}
|
||||
|
||||
const bool fast_k3x3 = kernel_type >= HTP_CONV2D_DW_KERNEL_WHCN_S1_K3X3;
|
||||
const uint32_t pad_rows = fast_k3x3 ? 4 : 0;
|
||||
const uint32_t pad_top_stride = fast_k3x3 ? w : 0;
|
||||
|
||||
const size_t vtcm_budget = (sess && htp_session_vtcm_size(sess) > 0) ? htp_session_vtcm_size(sess) : 0;
|
||||
if (vtcm_budget == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint32_t w_size = (uint32_t) (kw * kh * c * ggml_type_size(src0->type));
|
||||
|
||||
uint32_t max_threads_sess = (uint32_t) htp_session_n_threads(sess);
|
||||
uint32_t oh_chunk = oh_u;
|
||||
|
||||
uint32_t n_threads = max_threads_sess;
|
||||
struct htp_conv2d_dw_vtcm_layout layout;
|
||||
|
||||
while (true) {
|
||||
const uint32_t max_ih_chunk = (std::min)(h, (oh_chunk - 1) * (uint32_t) s1 + (kh - 1) * (uint32_t) d1 + 1);
|
||||
const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128;
|
||||
const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128;
|
||||
|
||||
htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads);
|
||||
|
||||
if (layout.total_bytes <= vtcm_budget && layout.total_bytes > 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (oh_chunk > 1) {
|
||||
oh_chunk = (oh_chunk + 1) / 2;
|
||||
} else if (n_threads > 1) {
|
||||
n_threads--;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t n_chunks = (oh_u + oh_chunk - 1) / oh_chunk;
|
||||
const uint32_t total_tasks = cwhn ? (n * n_chunks) : (c * n * n_chunks);
|
||||
n_threads = (std::min)(n_threads, total_tasks);
|
||||
if (n_threads == 0) {
|
||||
n_threads = 1;
|
||||
}
|
||||
|
||||
const uint32_t max_ih_chunk = (std::min)(h, (oh_chunk - 1) * (uint32_t) s1 + (kh - 1) * (uint32_t) d1 + 1);
|
||||
const uint32_t max_src_bytes = (max_ih_chunk + pad_rows) * row_src_bytes + 128;
|
||||
const uint32_t max_dst_bytes = oh_chunk * row_dst_bytes + 128;
|
||||
htp_conv2d_dw_vtcm_layout_build(&layout, max_src_bytes, max_dst_bytes, w_size, n_threads);
|
||||
|
||||
kparams->kernel_type = kernel_type;
|
||||
kparams->n_threads = (uint8_t) n_threads;
|
||||
kparams->cwhn = cwhn ? 1 : 0;
|
||||
kparams->is_f16 = (src0->type == GGML_TYPE_F16) ? 1 : 0;
|
||||
kparams->total_tasks = total_tasks;
|
||||
kparams->vtcm_size = (uint32_t) layout.total_bytes;
|
||||
kparams->w = w;
|
||||
kparams->h = h;
|
||||
kparams->c = c;
|
||||
kparams->n = n;
|
||||
kparams->kw = kw;
|
||||
kparams->kh = kh;
|
||||
kparams->ow = ow_u;
|
||||
kparams->oh = oh_u;
|
||||
kparams->s0 = s0;
|
||||
kparams->s1 = s1;
|
||||
kparams->p0 = p0;
|
||||
kparams->p1 = p1;
|
||||
kparams->d0 = d0;
|
||||
kparams->d1 = d1;
|
||||
kparams->oh_chunk = oh_chunk;
|
||||
kparams->n_chunks = n_chunks;
|
||||
kparams->row_src_bytes = row_src_bytes;
|
||||
kparams->row_dst_bytes = row_dst_bytes;
|
||||
kparams->pad_top_stride = pad_top_stride;
|
||||
kparams->div_c = init_fastdiv_values(c);
|
||||
kparams->div_n_chunks = init_fastdiv_values(n_chunks > 0 ? n_chunks : 1);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
template <auto SolveFn>
|
||||
static inline bool htp_kparams_invoke(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
@@ -4095,31 +3372,6 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode &
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (node.opcode == HTP_OP_UPSCALE) {
|
||||
if (node.inputs.empty() || !node.inputs[0] || node.outputs.empty() || !node.outputs[0]) {
|
||||
return;
|
||||
}
|
||||
const struct ggml_tensor * src0 = node.inputs[0];
|
||||
const struct ggml_tensor * dst = node.outputs[0];
|
||||
|
||||
if ((src0->ne[3] > 1 || dst->ne[3] > 1) &&
|
||||
src0->ne[2] == dst->ne[2] && src0->ne[3] == dst->ne[3] &&
|
||||
src0->nb[3] == (size_t) src0->ne[2] * src0->nb[2] &&
|
||||
dst->nb[3] == (size_t) dst->ne[2] * dst->nb[2]) {
|
||||
struct ggml_tensor src0_c = *src0;
|
||||
struct ggml_tensor dst_c = *dst;
|
||||
src0_c.ne[2] *= src0_c.ne[3];
|
||||
src0_c.ne[3] = 1;
|
||||
src0_c.nb[3] = (size_t) src0_c.ne[2] * src0_c.nb[2];
|
||||
dst_c.ne[2] *= dst_c.ne[3];
|
||||
dst_c.ne[3] = 1;
|
||||
dst_c.nb[3] = (size_t) dst_c.ne[2] * dst_c.nb[2];
|
||||
node.inputs[0] = node.add_dummy(src0_c);
|
||||
node.outputs[0] = node.add_dummy(dst_c);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
bool htp_kparams_solve(
|
||||
@@ -4149,9 +3401,6 @@ bool htp_kparams_solve(
|
||||
const size_t src2_size = src2 ? hex_round_up(ggml_nbytes(src2), 128) : 0;
|
||||
return htp_kparams_solve_matmul_impl(sess, src0, src1, dst, src2 ? src2->nb[1] : 0, src2_size, node.kparams<htp_mm_kernel_params>());
|
||||
}
|
||||
if (node.opcode == HTP_OP_UPSCALE) {
|
||||
return htp_kparams_solve_upscale(sess, src0, dst, node.kparams<htp_upscale_kernel_params>());
|
||||
}
|
||||
|
||||
return htp_kparams_solve(sess, node.node, node.kernel_params);
|
||||
}
|
||||
@@ -4162,7 +3411,7 @@ bool htp_kparams_solve(
|
||||
void * kparams
|
||||
) {
|
||||
const htp_op_code opcode = htp_op_remap(op);
|
||||
if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode) || opcode == HTP_OP_UPSCALE) {
|
||||
if (htp_op_is_unary(opcode) || htp_op_is_glu(opcode)) {
|
||||
htp_opnode tmp(opcode, op);
|
||||
if (!htp_kparams_solve(sess, tmp)) {
|
||||
return false;
|
||||
@@ -4227,12 +3476,6 @@ bool htp_kparams_solve(
|
||||
case HTP_OP_IM2COL_3D:
|
||||
return htp_kparams_invoke<htp_kparams_solve_im2col_3d>(sess, op, kparams);
|
||||
|
||||
case HTP_OP_CONV_2D:
|
||||
return htp_kparams_invoke<htp_kparams_solve_conv2d>(sess, op, kparams);
|
||||
|
||||
case HTP_OP_CONV_2D_DW:
|
||||
return htp_kparams_invoke<htp_kparams_solve_conv2d_dw>(sess, op, kparams);
|
||||
|
||||
case HTP_OP_SUM_ROWS:
|
||||
case HTP_OP_SUM:
|
||||
case HTP_OP_ARGMAX:
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
#include "htp/argsort-ops.h"
|
||||
#include "htp/binary-ops.h"
|
||||
#include "htp/concat-ops.h"
|
||||
#include "htp/conv2d-ops.h"
|
||||
#include "htp/cpy-ops.h"
|
||||
#include "htp/cumsum-ops.h"
|
||||
#include "htp/diag-ops.h"
|
||||
@@ -29,7 +28,6 @@
|
||||
#include "htp/ssm-conv.h"
|
||||
#include "htp/sum-rows-ops.h"
|
||||
#include "htp/unary-ops.h"
|
||||
#include "htp/upscale-ops.h"
|
||||
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
@@ -135,19 +133,4 @@ bool htp_kparams_solve_unary_mul_glu(
|
||||
struct htp_glu_kernel_params * kparams
|
||||
);
|
||||
|
||||
bool htp_kparams_solve_upscale(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * src0,
|
||||
const struct ggml_tensor * dst,
|
||||
struct htp_upscale_kernel_params * kparams
|
||||
);
|
||||
|
||||
static inline bool htp_kparams_solve_upscale(
|
||||
const struct ggml_hexagon_session * sess,
|
||||
const struct ggml_tensor * op,
|
||||
struct htp_upscale_kernel_params * kparams
|
||||
) {
|
||||
return htp_kparams_solve_upscale(sess, op->src[0], op, kparams);
|
||||
}
|
||||
|
||||
#endif // HTP_KPARAMS_H
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
#include "htp/ssm-conv.h"
|
||||
#include "htp/sum-rows-ops.h"
|
||||
#include "htp/unary-ops.h"
|
||||
#include "htp/upscale-ops.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
@@ -225,19 +224,6 @@ static std::string format_kernel_params(const htp_opnode & node) {
|
||||
} else if (node.opcode == HTP_OP_SOLVE_TRI) {
|
||||
const auto * kparams = node.kparams<htp_solve_tri_kernel_params>();
|
||||
snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads);
|
||||
} else if (node.opcode == HTP_OP_UPSCALE) {
|
||||
const auto * kparams = node.kparams<htp_upscale_kernel_params>();
|
||||
const char * ktype_str = "unknown";
|
||||
switch (kparams->kernel_type) {
|
||||
case HTP_UPSCALE_KERNEL_NEAREST: ktype_str = "nearest"; break;
|
||||
case HTP_UPSCALE_KERNEL_NEAREST_2X: ktype_str = "nearest-2x"; break;
|
||||
case HTP_UPSCALE_KERNEL_NEAREST_4X: ktype_str = "nearest-4x"; break;
|
||||
case HTP_UPSCALE_KERNEL_BILINEAR: ktype_str = kparams->align_corners ? "bilinear-ac" : "bilinear"; break;
|
||||
case HTP_UPSCALE_KERNEL_BILINEAR_2X: ktype_str = "bilinear-2x"; break;
|
||||
case HTP_UPSCALE_KERNEL_BILINEAR_AA: ktype_str = "bilinear-aa"; break;
|
||||
case HTP_UPSCALE_KERNEL_BICUBIC: ktype_str = "bicubic"; break;
|
||||
}
|
||||
snprintf(str, max_size, "%s nth %d chk %d vtcm %u", ktype_str, (int) kparams->n_threads, (int) kparams->dst_nrows_per_chunk, (unsigned int) kparams->vtcm_size);
|
||||
} else {
|
||||
snprintf(str, max_size, "----");
|
||||
}
|
||||
|
||||
@@ -44,10 +44,8 @@ add_library(${HTP_LIB} SHARED
|
||||
pad-ops.c
|
||||
argsort-ops.c
|
||||
im2col-ops.c
|
||||
conv2d-ops.c
|
||||
roll-ops.c
|
||||
pool-ops.c
|
||||
upscale-ops.c
|
||||
allreduce-ops.c
|
||||
)
|
||||
|
||||
|
||||
@@ -47,108 +47,87 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
|
||||
const uint32_t src0_ne0 = src0->ne[0];
|
||||
const uint32_t src1_ne0 = src1->ne[0];
|
||||
|
||||
// dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column
|
||||
const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread;
|
||||
const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows);
|
||||
if (row_begin >= row_end) return;
|
||||
const uint32_t row_end = cctx->row_start + cctx->nrows;
|
||||
const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
|
||||
const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
|
||||
if (start_i >= end_i) return;
|
||||
|
||||
dma_queue * dma_q = octx->ctx->dma[ith];
|
||||
|
||||
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
|
||||
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
|
||||
|
||||
const uint32_t block_i = 32;
|
||||
const uint32_t spad1_stride = block_i * sizeof(float);
|
||||
|
||||
const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride);
|
||||
const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 32);
|
||||
const uint32_t src0_row_bytes = src0_ne0 * sizeof(float);
|
||||
const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
|
||||
const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
|
||||
const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(float);
|
||||
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
|
||||
|
||||
const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
|
||||
const uint32_t ne2 = dst->ne[2];
|
||||
|
||||
// tile: block_rows dst rows x block_cols src1 columns
|
||||
const uint32_t block_rows = 128;
|
||||
const uint32_t block_cols = 256;
|
||||
// +1 element spreads the gather lanes across VTCM, so they do not serialize on one path
|
||||
const uint32_t spad1_stride = block_rows * sizeof(float) + sizeof(float);
|
||||
uint32_t p = 0;
|
||||
uint32_t i = start_i;
|
||||
|
||||
const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride);
|
||||
const uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, 32), block_cols);
|
||||
const uint32_t src0_row_bytes = src0_ne0 * sizeof(float);
|
||||
const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
|
||||
const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
|
||||
const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(float);
|
||||
const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(float), src1_ne0);
|
||||
|
||||
// two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other
|
||||
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
|
||||
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
|
||||
const uint32_t spad0_slot = block_rows * spad0_stride;
|
||||
const uint32_t spad1_slot = block_cols_padded * spad1_stride;
|
||||
const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
|
||||
|
||||
// tiles walk (plane, row, col) with col fastest; src0 goes out with the col == 0 tile only
|
||||
uint32_t plane = 0;
|
||||
uint32_t row = row_begin;
|
||||
uint32_t col = 0;
|
||||
uint32_t slot = 0;
|
||||
bool dst_pending = false;
|
||||
dma_queue_pop(dma_q); // src1
|
||||
|
||||
const dma_addr_t src1_addr = concat_plane_addr(src1, plane, div_ne2, ne2) + row * src1->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(row_end - row, block_rows) * sizeof(float), MIN(src1_ne0, block_cols));
|
||||
while (p < cctx->nplanes) {
|
||||
const uint32_t current_block_i = MIN(end_i - i, block_i);
|
||||
|
||||
const dma_addr_t src0_addr = concat_plane_addr(src0, plane, div_ne2, ne2) + row * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_stride, src0->nb[1], src0_row_bytes, MIN(row_end - row, block_rows));
|
||||
|
||||
while (plane < cctx->nplanes) {
|
||||
const uint32_t nrows = MIN(row_end - row, block_rows);
|
||||
const uint32_t ncols = MIN(src1_ne0 - col, block_cols);
|
||||
|
||||
uint8_t * spad0 = spad0_base + slot * spad0_slot;
|
||||
uint8_t * spad1 = spad1_base + slot * spad1_slot;
|
||||
|
||||
dma_queue_pop(dma_q); // src1
|
||||
if (col == 0) {
|
||||
dma_queue_pop(dma_q); // src0
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
|
||||
for (uint32_t j = 0; j < src1_ne0; j += 32) {
|
||||
const uint8_t * src_ptr = spad1_base + j * spad1_stride;
|
||||
uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(float);
|
||||
hvx_transpose_32x32_w_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 32));
|
||||
}
|
||||
hvx_gather_sync(spad0_base + src0_row_padded);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
|
||||
|
||||
uint32_t next_plane = plane;
|
||||
uint32_t next_row = row;
|
||||
uint32_t next_col = col + block_cols;
|
||||
if (next_col >= src1_ne0) {
|
||||
next_col = 0;
|
||||
next_row = row + block_rows;
|
||||
if (next_row >= row_end) {
|
||||
next_row = row_begin;
|
||||
next_plane++;
|
||||
}
|
||||
uint32_t np = p;
|
||||
uint32_t ni = i + block_i;
|
||||
if (ni >= end_i) {
|
||||
ni = start_i;
|
||||
np++;
|
||||
}
|
||||
const bool has_next = next_plane < cctx->nplanes;
|
||||
const uint32_t next_nrows = MIN(row_end - next_row, block_rows);
|
||||
const uint32_t next_ncols = MIN(src1_ne0 - next_col, block_cols);
|
||||
const bool has_next = np < cctx->nplanes;
|
||||
const uint32_t next_block_i = MIN(end_i - ni, block_i);
|
||||
|
||||
// other spad1 slot is free since the previous gather sync
|
||||
// spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
|
||||
if (has_next) {
|
||||
const dma_addr_t addr = concat_plane_addr(src1, next_plane, div_ne2, ne2) + next_row * src1->nb[1] + next_col * src1->nb[0];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base + (slot ^ 1) * spad1_slot, addr), spad1_stride, src1->nb[0], next_nrows * sizeof(float), next_ncols);
|
||||
const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(float), src1_ne0);
|
||||
}
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
|
||||
for (uint32_t c = 0; c < ncols; c += 32) {
|
||||
hvx_transpose_32x32_w_gather(spad0 + src0_row_padded + c * sizeof(float), spad0_stride, spad1 + c * spad1_stride, spad1_stride, offsets, nrows, MIN(ncols - c, 32));
|
||||
}
|
||||
hvx_gather_sync(spad0 + src0_row_padded);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
|
||||
dma_queue_pop(dma_q); // src0
|
||||
|
||||
// other spad0 slot is free once dst[b-1] is written
|
||||
if (dst_pending) {
|
||||
dma_queue_pop(dma_q); // dst
|
||||
const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i);
|
||||
|
||||
if (has_next) {
|
||||
dma_queue_pop(dma_q); // next src1
|
||||
}
|
||||
if (has_next && next_col == 0) {
|
||||
const dma_addr_t addr = concat_plane_addr(src0, next_plane, div_ne2, ne2) + next_row * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + (slot ^ 1) * spad0_slot + src0_pre, addr), spad0_stride, src0->nb[1], src0_row_bytes, next_nrows);
|
||||
dma_queue_pop(dma_q); // dst
|
||||
|
||||
// spad0 is free after the dst write
|
||||
if (has_next) {
|
||||
const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
|
||||
}
|
||||
|
||||
// the col == 0 tile also carries the src0 columns in front
|
||||
const uint32_t lead = (col == 0) ? src0_row_bytes : 0;
|
||||
const dma_addr_t dst_addr = concat_plane_addr(dst, plane, div_ne2, ne2) + row * dst->nb[1] + src0_row_bytes + col * sizeof(float) - lead;
|
||||
dma_queue_push(dma_q, dma_make_data(dst_addr, spad0 + src0_row_padded - lead), dst->nb[1], spad0_stride, lead + ncols * sizeof(float), nrows);
|
||||
dst_pending = true;
|
||||
|
||||
plane = next_plane;
|
||||
row = next_row;
|
||||
col = next_col;
|
||||
slot ^= 1;
|
||||
p = np;
|
||||
i = ni;
|
||||
}
|
||||
dma_queue_flush(dma_q);
|
||||
}
|
||||
@@ -164,108 +143,87 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
|
||||
const uint32_t src0_ne0 = src0->ne[0];
|
||||
const uint32_t src1_ne0 = src1->ne[0];
|
||||
|
||||
// dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column
|
||||
const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread;
|
||||
const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows);
|
||||
if (row_begin >= row_end) return;
|
||||
const uint32_t row_end = cctx->row_start + cctx->nrows;
|
||||
const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
|
||||
const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
|
||||
if (start_i >= end_i) return;
|
||||
|
||||
dma_queue * dma_q = octx->ctx->dma[ith];
|
||||
|
||||
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
|
||||
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
|
||||
|
||||
const uint32_t block_i = 64;
|
||||
const uint32_t spad1_stride = block_i * sizeof(__fp16);
|
||||
|
||||
const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride);
|
||||
const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 64);
|
||||
const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16);
|
||||
const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
|
||||
const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
|
||||
const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(__fp16);
|
||||
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
|
||||
|
||||
const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
|
||||
const uint32_t ne2 = dst->ne[2];
|
||||
|
||||
// tile: block_rows dst rows x block_cols src1 columns
|
||||
const uint32_t block_rows = 128;
|
||||
const uint32_t block_cols = 512;
|
||||
// +1 element spreads the gather lanes across VTCM, so they do not serialize on one path
|
||||
const uint32_t spad1_stride = block_rows * sizeof(__fp16) + sizeof(__fp16);
|
||||
uint32_t p = 0;
|
||||
uint32_t i = start_i;
|
||||
|
||||
const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride);
|
||||
const uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, 64), block_cols);
|
||||
const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16);
|
||||
const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
|
||||
const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
|
||||
const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(__fp16);
|
||||
const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(__fp16), src1_ne0);
|
||||
|
||||
// two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other
|
||||
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
|
||||
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
|
||||
const uint32_t spad0_slot = block_rows * spad0_stride;
|
||||
const uint32_t spad1_slot = block_cols_padded * spad1_stride;
|
||||
const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
|
||||
|
||||
// tiles walk (plane, row, col) with col fastest; src0 goes out with the col == 0 tile only
|
||||
uint32_t plane = 0;
|
||||
uint32_t row = row_begin;
|
||||
uint32_t col = 0;
|
||||
uint32_t slot = 0;
|
||||
bool dst_pending = false;
|
||||
dma_queue_pop(dma_q); // src1
|
||||
|
||||
const dma_addr_t src1_addr = concat_plane_addr(src1, plane, div_ne2, ne2) + row * src1->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(row_end - row, block_rows) * sizeof(__fp16), MIN(src1_ne0, block_cols));
|
||||
while (p < cctx->nplanes) {
|
||||
const uint32_t current_block_i = MIN(end_i - i, block_i);
|
||||
|
||||
const dma_addr_t src0_addr = concat_plane_addr(src0, plane, div_ne2, ne2) + row * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_stride, src0->nb[1], src0_row_bytes, MIN(row_end - row, block_rows));
|
||||
|
||||
while (plane < cctx->nplanes) {
|
||||
const uint32_t nrows = MIN(row_end - row, block_rows);
|
||||
const uint32_t ncols = MIN(src1_ne0 - col, block_cols);
|
||||
|
||||
uint8_t * spad0 = spad0_base + slot * spad0_slot;
|
||||
uint8_t * spad1 = spad1_base + slot * spad1_slot;
|
||||
|
||||
dma_queue_pop(dma_q); // src1
|
||||
if (col == 0) {
|
||||
dma_queue_pop(dma_q); // src0
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
|
||||
for (uint32_t j = 0; j < src1_ne0; j += 64) {
|
||||
const uint8_t * src_ptr = spad1_base + j * spad1_stride;
|
||||
uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(__fp16);
|
||||
hvx_transpose_64x64_h_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 64));
|
||||
}
|
||||
hvx_gather_sync(spad0_base + src0_row_padded);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
|
||||
|
||||
uint32_t next_plane = plane;
|
||||
uint32_t next_row = row;
|
||||
uint32_t next_col = col + block_cols;
|
||||
if (next_col >= src1_ne0) {
|
||||
next_col = 0;
|
||||
next_row = row + block_rows;
|
||||
if (next_row >= row_end) {
|
||||
next_row = row_begin;
|
||||
next_plane++;
|
||||
}
|
||||
uint32_t np = p;
|
||||
uint32_t ni = i + block_i;
|
||||
if (ni >= end_i) {
|
||||
ni = start_i;
|
||||
np++;
|
||||
}
|
||||
const bool has_next = next_plane < cctx->nplanes;
|
||||
const uint32_t next_nrows = MIN(row_end - next_row, block_rows);
|
||||
const uint32_t next_ncols = MIN(src1_ne0 - next_col, block_cols);
|
||||
const bool has_next = np < cctx->nplanes;
|
||||
const uint32_t next_block_i = MIN(end_i - ni, block_i);
|
||||
|
||||
// other spad1 slot is free since the previous gather sync
|
||||
// spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
|
||||
if (has_next) {
|
||||
const dma_addr_t addr = concat_plane_addr(src1, next_plane, div_ne2, ne2) + next_row * src1->nb[1] + next_col * src1->nb[0];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base + (slot ^ 1) * spad1_slot, addr), spad1_stride, src1->nb[0], next_nrows * sizeof(__fp16), next_ncols);
|
||||
const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(__fp16), src1_ne0);
|
||||
}
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
|
||||
for (uint32_t c = 0; c < ncols; c += 64) {
|
||||
hvx_transpose_64x64_h_gather(spad0 + src0_row_padded + c * sizeof(__fp16), spad0_stride, spad1 + c * spad1_stride, spad1_stride, offsets, nrows, MIN(ncols - c, 64));
|
||||
}
|
||||
hvx_gather_sync(spad0 + src0_row_padded);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
|
||||
dma_queue_pop(dma_q); // src0
|
||||
|
||||
// other spad0 slot is free once dst[b-1] is written
|
||||
if (dst_pending) {
|
||||
dma_queue_pop(dma_q); // dst
|
||||
const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i);
|
||||
|
||||
if (has_next) {
|
||||
dma_queue_pop(dma_q); // next src1
|
||||
}
|
||||
if (has_next && next_col == 0) {
|
||||
const dma_addr_t addr = concat_plane_addr(src0, next_plane, div_ne2, ne2) + next_row * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + (slot ^ 1) * spad0_slot + src0_pre, addr), spad0_stride, src0->nb[1], src0_row_bytes, next_nrows);
|
||||
dma_queue_pop(dma_q); // dst
|
||||
|
||||
// spad0 is free after the dst write
|
||||
if (has_next) {
|
||||
const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
|
||||
}
|
||||
|
||||
// the col == 0 tile also carries the src0 columns in front
|
||||
const uint32_t lead = (col == 0) ? src0_row_bytes : 0;
|
||||
const dma_addr_t dst_addr = concat_plane_addr(dst, plane, div_ne2, ne2) + row * dst->nb[1] + src0_row_bytes + col * sizeof(__fp16) - lead;
|
||||
dma_queue_push(dma_q, dma_make_data(dst_addr, spad0 + src0_row_padded - lead), dst->nb[1], spad0_stride, lead + ncols * sizeof(__fp16), nrows);
|
||||
dst_pending = true;
|
||||
|
||||
plane = next_plane;
|
||||
row = next_row;
|
||||
col = next_col;
|
||||
slot ^= 1;
|
||||
p = np;
|
||||
i = ni;
|
||||
}
|
||||
dma_queue_flush(dma_q);
|
||||
}
|
||||
|
||||
@@ -53,17 +53,13 @@ static inline void htp_concat_transposed_vtcm_layout_build(
|
||||
uint32_t type_size,
|
||||
uint32_t n_threads) {
|
||||
|
||||
// src1_ne0 is processed in chunks of block_cols tokens, so VTCM does not grow with the batch
|
||||
uint32_t block_rows = 128;
|
||||
uint32_t block_cols = (type_size == 4) ? 256 : 512;
|
||||
// +1 element spreads the gather lanes across VTCM, so they do not serialize on one path
|
||||
uint32_t spad1_stride = block_rows * type_size + type_size;
|
||||
uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, (type_size == 4) ? 32 : 64), block_cols);
|
||||
uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + block_cols_padded * type_size;
|
||||
uint32_t block_i = (type_size == 4) ? 32 : 64;
|
||||
uint32_t spad1_stride = block_i * type_size;
|
||||
uint32_t src1_ne0_padded = hex_round_up(src1_ne0, block_i);
|
||||
uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + src1_ne0_padded * type_size;
|
||||
|
||||
// double-buffered: two slots per thread
|
||||
L->src0_spad_size_per_thread = 2 * block_rows * spad0_row_bytes;
|
||||
L->src1_spad_size_per_thread = 2 * block_cols_padded * spad1_stride;
|
||||
L->src0_spad_size_per_thread = block_i * spad0_row_bytes;
|
||||
L->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;
|
||||
|
||||
size_t off = 0;
|
||||
VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,197 +0,0 @@
|
||||
#ifndef HTP_CONV2D_OPS_H
|
||||
#define HTP_CONV2D_OPS_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "hex-fastdiv.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-vtcm.h"
|
||||
|
||||
enum htp_conv2d_kernel_type {
|
||||
HTP_CONV2D_KERNEL_UNSUPPORTED = 0,
|
||||
HTP_CONV2D_KERNEL_HMX_WINDOW, // HMX convolution with sliding-window staging
|
||||
HTP_CONV2D_KERNEL_HMX_TILED, // tiled implicit-GEMM convolution on HMX
|
||||
};
|
||||
|
||||
// HMX_WINDOW output staging buffers per thread
|
||||
#define HTP_CONV2D_WIN_OBUF 2
|
||||
|
||||
// HMX scales of one row tile at the start of VTCM: 32 (scale, bias) fp16 pairs and 128 zero bytes
|
||||
#define HTP_CONV2D_HMX_SCALES_SIZE 256
|
||||
|
||||
struct htp_conv2d_tiled_params {
|
||||
uint32_t w_kstride;
|
||||
uint32_t k_size;
|
||||
uint32_t k_group;
|
||||
uint32_t k_group_pad;
|
||||
uint32_t n_groups;
|
||||
uint32_t n_k_tiles;
|
||||
uint32_t n_ow_tiles;
|
||||
uint32_t seg_tiles;
|
||||
uint32_t stage_row;
|
||||
uint32_t n_stage;
|
||||
uint32_t wk_chunk;
|
||||
struct fastdiv_values div_ow_tiles;
|
||||
struct fastdiv_values div_oh;
|
||||
};
|
||||
|
||||
struct htp_conv2d_window_params {
|
||||
uint32_t fold;
|
||||
uint32_t n_cc;
|
||||
uint32_t act_rt_last;
|
||||
uint32_t sr;
|
||||
uint32_t ihs;
|
||||
uint32_t max_off;
|
||||
uint32_t n_bc;
|
||||
uint32_t seg_b;
|
||||
uint32_t seg_t;
|
||||
uint32_t w_cs;
|
||||
struct fastdiv_values div_sr;
|
||||
struct fastdiv_values div_ihs;
|
||||
struct fastdiv_values div_kw;
|
||||
struct fastdiv_values div_tg;
|
||||
struct fastdiv_values div_cc;
|
||||
struct fastdiv_values div_cs;
|
||||
};
|
||||
|
||||
struct htp_conv2d_kernel_params {
|
||||
uint8_t kernel_type;
|
||||
uint8_t n_threads;
|
||||
uint8_t cwhn;
|
||||
uint8_t w_kmajor;
|
||||
|
||||
uint32_t total_rows;
|
||||
uint32_t n_row_tiles;
|
||||
uint32_t ct_chunk;
|
||||
uint32_t rt_block;
|
||||
uint32_t stage_size;
|
||||
uint32_t thr_size;
|
||||
uint32_t vtcm_size;
|
||||
|
||||
union {
|
||||
struct htp_conv2d_tiled_params tiled;
|
||||
struct htp_conv2d_window_params window;
|
||||
};
|
||||
};
|
||||
|
||||
#if defined(__cplusplus)
|
||||
static_assert(sizeof(struct htp_conv2d_kernel_params) <= 128, "htp_conv2d_kernel_params is too large for kernel_params blob");
|
||||
#else
|
||||
_Static_assert(sizeof(struct htp_conv2d_kernel_params) <= 128, "htp_conv2d_kernel_params is too large for kernel_params blob");
|
||||
#endif
|
||||
|
||||
enum htp_conv2d_dw_kernel_type {
|
||||
HTP_CONV2D_DW_KERNEL_UNSUPPORTED = 0,
|
||||
HTP_CONV2D_DW_KERNEL_CWHN, // CWHN channel-contiguous layout
|
||||
HTP_CONV2D_DW_KERNEL_WHCN_S1, // WHCN general stride-1
|
||||
HTP_CONV2D_DW_KERNEL_WHCN_S2, // WHCN general stride-2
|
||||
HTP_CONV2D_DW_KERNEL_WHCN_S1_K3X3, // WHCN aligned w%32==0, 3x3 s=1 p=1 d=1
|
||||
HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W32, // WHCN w=32->ow=16, 3x3 s=2 p=1 d=1 (2 rows per vector)
|
||||
HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3_W64, // WHCN w=64->ow=32, 3x3 s=2 p=1 d=1 (1 vector rolling kh)
|
||||
HTP_CONV2D_DW_KERNEL_WHCN_S2_K3X3, // WHCN aligned w%32==0 (w>=96), 3x3 s=2 p=1 d=1
|
||||
HTP_CONV2D_DW_KERNEL_COUNT,
|
||||
};
|
||||
|
||||
struct htp_conv2d_dw_kernel_params {
|
||||
uint8_t kernel_type;
|
||||
uint8_t n_threads;
|
||||
uint8_t cwhn;
|
||||
uint8_t is_f16;
|
||||
|
||||
uint32_t total_tasks;
|
||||
uint32_t vtcm_size;
|
||||
uint32_t w;
|
||||
uint32_t h;
|
||||
uint32_t c;
|
||||
uint32_t n;
|
||||
uint32_t kw;
|
||||
uint32_t kh;
|
||||
uint32_t ow;
|
||||
uint32_t oh;
|
||||
int32_t s0;
|
||||
int32_t s1;
|
||||
int32_t p0;
|
||||
int32_t p1;
|
||||
int32_t d0;
|
||||
int32_t d1;
|
||||
uint32_t oh_chunk;
|
||||
uint32_t n_chunks;
|
||||
uint32_t row_src_bytes;
|
||||
uint32_t row_dst_bytes;
|
||||
uint32_t pad_top_stride;
|
||||
struct fastdiv_values div_c;
|
||||
struct fastdiv_values div_n_chunks;
|
||||
};
|
||||
|
||||
#if defined(__cplusplus)
|
||||
static_assert(sizeof(struct htp_conv2d_dw_kernel_params) <= 128, "htp_conv2d_dw_kernel_params is too large for kernel_params blob");
|
||||
#else
|
||||
_Static_assert(sizeof(struct htp_conv2d_dw_kernel_params) <= 128, "htp_conv2d_dw_kernel_params is too large for kernel_params blob");
|
||||
#endif
|
||||
|
||||
struct htp_conv2d_dw_vtcm_layout {
|
||||
size_t total_bytes;
|
||||
size_t off_w;
|
||||
size_t off_src;
|
||||
size_t off_dst;
|
||||
size_t src_spad_half_size;
|
||||
size_t dst_spad_half_size;
|
||||
size_t src_bytes_per_thread;
|
||||
size_t dst_bytes_per_thread;
|
||||
};
|
||||
|
||||
static inline void htp_conv2d_dw_vtcm_layout_build(
|
||||
struct htp_conv2d_dw_vtcm_layout * L,
|
||||
uint32_t max_src_bytes,
|
||||
uint32_t max_dst_bytes,
|
||||
uint32_t w_size,
|
||||
uint32_t n_threads
|
||||
) {
|
||||
L->src_spad_half_size = hex_round_up(max_src_bytes + 256, 128);
|
||||
L->dst_spad_half_size = hex_round_up(max_dst_bytes + 128, 128);
|
||||
L->src_bytes_per_thread = 2 * L->src_spad_half_size;
|
||||
L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
|
||||
|
||||
size_t off = 0;
|
||||
VTCM_LAYOUT_ALLOC(off, off_w, hex_round_up(w_size, 128));
|
||||
VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
|
||||
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
|
||||
L->total_bytes = off;
|
||||
}
|
||||
|
||||
struct htp_conv2d_vtcm_layout {
|
||||
size_t total_bytes;
|
||||
size_t off_scales;
|
||||
size_t off_w;
|
||||
size_t off_act[2];
|
||||
size_t off_out[2];
|
||||
size_t off_thr;
|
||||
|
||||
size_t act_size;
|
||||
};
|
||||
|
||||
static inline void htp_conv2d_vtcm_layout_build(
|
||||
struct htp_conv2d_vtcm_layout * L,
|
||||
uint32_t rt_block,
|
||||
size_t w_size,
|
||||
size_t act_size,
|
||||
size_t out_size,
|
||||
size_t thr_size,
|
||||
uint32_t n_threads
|
||||
) {
|
||||
const size_t sc = hex_round_up((size_t) rt_block * HTP_CONV2D_HMX_SCALES_SIZE, 2048);
|
||||
size_t off = 0;
|
||||
VTCM_LAYOUT_ALLOC(off, off_scales, sc);
|
||||
VTCM_LAYOUT_ALLOC(off, off_w, w_size);
|
||||
VTCM_LAYOUT_ALLOC(off, off_act[0], act_size);
|
||||
VTCM_LAYOUT_ALLOC(off, off_act[1], act_size);
|
||||
VTCM_LAYOUT_ALLOC(off, off_out[0], out_size);
|
||||
VTCM_LAYOUT_ALLOC(off, off_out[1], out_size);
|
||||
VTCM_LAYOUT_ALLOC(off, off_thr, (size_t) n_threads * thr_size);
|
||||
L->act_size = act_size;
|
||||
L->total_bytes = off;
|
||||
}
|
||||
|
||||
#endif // HTP_CONV2D_OPS_H
|
||||
@@ -1456,10 +1456,9 @@ static inline void gdn_dma_push_chunk_gb(
|
||||
const dma_addr_t g_dma = g->data + (uint64_t) iv3 * g->nb[3] + (uint64_t) t_chunk * g->nb[2] + (uint64_t) iv1 * g->nb[1];
|
||||
const dma_addr_t beta_dma = beta->data + (uint64_t) iv3 * beta->nb[3] + (uint64_t) t_chunk * beta->nb[2] + (uint64_t) iv1 * beta->nb[1];
|
||||
const uint32_t row_bytes = n_batch * sizeof(float);
|
||||
const uint32_t dst_stride = (n_batch > 1) ? row_bytes + sizeof(float) : row_bytes;
|
||||
|
||||
dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), dst_stride, g->nb[2], row_bytes, chunk_size);
|
||||
dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), dst_stride, beta->nb[2], row_bytes, chunk_size);
|
||||
dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), row_bytes, g->nb[2], row_bytes, chunk_size);
|
||||
dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), row_bytes, beta->nb[2], row_bytes, chunk_size);
|
||||
}
|
||||
|
||||
static inline void gdn_pack_s_col_tiles(
|
||||
@@ -1708,16 +1707,19 @@ static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data)
|
||||
hvx_vmem(head->b_f32[curr_buf] + 0) = hvx_vmem(bctx->vtcm_b_raw + 0);
|
||||
hvx_vmem(head->b_f32[curr_buf] + 32) = hvx_vmem(bctx->vtcm_b_raw + 32);
|
||||
} else {
|
||||
const uint32_t vtcm_stride = (n_batch + 1) * sizeof(float);
|
||||
const HVX_Vector vv = hvx_vec_gather_offsets_w(vtcm_stride);
|
||||
int32_t offsets[32] __attribute__((aligned(128)));
|
||||
for (int k = 0; k < 32; ++k) {
|
||||
offsets[k] = k * n_batch * sizeof(float);
|
||||
}
|
||||
HVX_Vector vv = *(const HVX_Vector *) offsets;
|
||||
const size_t rt_g = (size_t) ((const uint8_t *) bctx->vtcm_g_raw + i * sizeof(float));
|
||||
const size_t rt_b = (size_t) ((const uint8_t *) bctx->vtcm_b_raw + i * sizeof(float));
|
||||
const size_t mu = 64 * vtcm_stride;
|
||||
const size_t mu = 64 * n_batch * sizeof(float);
|
||||
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 0), rt_g, mu, vv);
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * vtcm_stride, mu, vv);
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * n_batch * sizeof(float), mu, vv);
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 0), rt_b, mu, vv);
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * vtcm_stride, mu, vv);
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * n_batch * sizeof(float), mu, vv);
|
||||
}
|
||||
|
||||
const uint32_t t_chunk = bctx->c * 64;
|
||||
|
||||
@@ -204,7 +204,7 @@ static inline void htp_gdn_hmx_vtcm_layout_build(
|
||||
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_f32[1], bh * dma_scalar_sz, pipeline);
|
||||
VTCM_LAYOUT_ALLOC(off, off_b_f32[0], bh * dma_scalar_sz);
|
||||
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_f32[1], bh * dma_scalar_sz, pipeline);
|
||||
const size_t raw_gb_sz = hex_round_up((bh + 1) * chunk_size * sizeof(float), 128);
|
||||
const size_t raw_gb_sz = hex_round_up(bh * chunk_size * sizeof(float), 128);
|
||||
VTCM_LAYOUT_ALLOC(off, off_g_raw[0], raw_gb_sz);
|
||||
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_raw[1], raw_gb_sz, pipeline);
|
||||
VTCM_LAYOUT_ALLOC(off, off_b_raw[0], raw_gb_sz);
|
||||
|
||||
@@ -182,12 +182,9 @@ int op_gated_delta_net(struct htp_ops_context * octx);
|
||||
int op_pad(struct htp_ops_context * octx);
|
||||
int op_im2col(struct htp_ops_context * octx);
|
||||
int op_im2col_3d(struct htp_ops_context * octx);
|
||||
int op_conv_2d(struct htp_ops_context * octx);
|
||||
int op_conv_2d_dw(struct htp_ops_context * octx);
|
||||
int op_pool_2d(struct htp_ops_context * octx);
|
||||
int op_pool_1d(struct htp_ops_context * octx);
|
||||
int op_allreduce(struct htp_ops_context * octx);
|
||||
int op_roll(struct htp_ops_context * octx);
|
||||
int op_upscale(struct htp_ops_context * octx);
|
||||
|
||||
#endif /* HTP_CTX_H */
|
||||
|
||||
@@ -96,8 +96,6 @@ enum htp_op_code {
|
||||
HTP_OP_CLAMP,
|
||||
HTP_OP_LEAKY_RELU,
|
||||
HTP_OP_IM2COL,
|
||||
HTP_OP_CONV_2D,
|
||||
HTP_OP_CONV_2D_DW,
|
||||
HTP_OP_FENCE,
|
||||
HTP_OP_ALLREDUCE,
|
||||
HTP_OP_ALLREDUCE_ADD,
|
||||
@@ -116,7 +114,6 @@ enum htp_op_code {
|
||||
HTP_OP_GLU_SIGMOID,
|
||||
HTP_OP_UNARY_CEIL,
|
||||
HTP_OP_UNARY_SGN,
|
||||
HTP_OP_UPSCALE,
|
||||
|
||||
HTP_OP_INVALID
|
||||
};
|
||||
@@ -137,8 +134,6 @@ enum htp_op_code {
|
||||
|
||||
#define HTP_MMAP_MAX_VMEM (2147483648u)
|
||||
|
||||
#define HTP_DDR_SPAD_SIZE (512 * 1024)
|
||||
|
||||
enum htp_tensor_flags {
|
||||
HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)
|
||||
HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format
|
||||
|
||||
@@ -26,7 +26,6 @@ struct htp_im2col_context {
|
||||
uint32_t pe_row_base; // first output row index assigned to this dev (DMA path)
|
||||
uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path)
|
||||
uint32_t pe_rows_per_thread; // output rows per worker
|
||||
uint32_t pe_blocks_per_thread; // output-col blocks per worker (blocked DMA path)
|
||||
uint32_t pe_src_row_bytes; // one output row's source, rounded 256
|
||||
uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256
|
||||
|
||||
@@ -39,7 +38,6 @@ struct htp_im2col_context {
|
||||
uint32_t pe_owb; // output-col block size
|
||||
uint32_t pe_wb; // staged source window width
|
||||
uint32_t pe_nbpr; // blocks per row
|
||||
uint32_t pe_gather_nvec; // vectors per gathered patch (KH == 1 blocked path)
|
||||
|
||||
// Precomputed dimensions for worker threads
|
||||
uint32_t N;
|
||||
@@ -53,7 +51,6 @@ struct htp_im2col_context {
|
||||
uint32_t thread_in[HTP_MAX_NTHREADS];
|
||||
uint32_t thread_iod[HTP_MAX_NTHREADS];
|
||||
uint32_t thread_ioh[HTP_MAX_NTHREADS];
|
||||
uint32_t thread_c0[HTP_MAX_NTHREADS];
|
||||
|
||||
void * compute;
|
||||
uint32_t dst_elem;
|
||||
@@ -218,109 +215,6 @@ static void FNAME(
|
||||
IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u)
|
||||
IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u)
|
||||
|
||||
// Offsets of one output patch in the staged window, for KH == 1: lane k reads channel k / KW at tap k % KW.
|
||||
// Lanes past the patch read the first element and are not stored.
|
||||
// The first vector is built lane by lane; each next one is 32 lanes further: the tap advances by 32 % KW
|
||||
// and the channel by 32 / KW, plus one more when the tap wraps past KW.
|
||||
static __attribute__((noinline)) void im2col_gather_table_build(const struct htp_im2col_context * ictx,
|
||||
HVX_Vector * restrict table) {
|
||||
const int32_t d0 = ictx->octx->op_params[4];
|
||||
const uint32_t KW = ictx->KW, Wb = ictx->pe_wb;
|
||||
|
||||
int32_t tap0[32] __attribute__((aligned(128)));
|
||||
int32_t off0[32] __attribute__((aligned(128)));
|
||||
uint32_t iic = 0, ikw = 0;
|
||||
for (uint32_t k = 0; k < 32; k++) {
|
||||
tap0[k] = (int32_t) ikw;
|
||||
off0[k] = (int32_t) ((iic * Wb + ikw * d0) * sizeof(float));
|
||||
if (++ikw == KW) {
|
||||
ikw = 0;
|
||||
iic++;
|
||||
}
|
||||
}
|
||||
// (iic, ikw) is now (32 / KW, 32 % KW)
|
||||
const HVX_Vector v_kw = Q6_V_vsplat_R(KW);
|
||||
const HVX_Vector v_kw_m1 = Q6_V_vsplat_R(KW - 1);
|
||||
const HVX_Vector v_tap_step = Q6_V_vsplat_R(ikw);
|
||||
const HVX_Vector v_off_step = Q6_V_vsplat_R((iic * Wb + ikw * d0) * sizeof(float));
|
||||
const HVX_Vector v_off_wrap = Q6_V_vsplat_R((Wb - KW * d0) * sizeof(float));
|
||||
const HVX_Vector v_32 = Q6_V_vsplat_R(32);
|
||||
const HVX_Vector v_n = Q6_V_vsplat_R(ictx->patch_stride);
|
||||
|
||||
HVX_Vector lane = hvx_vec_gather_offsets_w(1);
|
||||
HVX_Vector tap = *(HVX_Vector *) tap0;
|
||||
HVX_Vector off = *(HVX_Vector *) off0;
|
||||
for (uint32_t g = 0; g < ictx->pe_gather_nvec; g++) {
|
||||
table[g] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(v_n, lane), off, Q6_V_vzero());
|
||||
|
||||
lane = Q6_Vw_vadd_VwVw(lane, v_32);
|
||||
tap = Q6_Vw_vadd_VwVw(tap, v_tap_step);
|
||||
off = Q6_Vw_vadd_VwVw(off, v_off_step);
|
||||
|
||||
const HVX_VectorPred wrap = Q6_Q_vcmp_gt_VwVw(tap, v_kw_m1);
|
||||
tap = Q6_V_vmux_QVV(wrap, Q6_Vw_vsub_VwVw(tap, v_kw), tap);
|
||||
off = Q6_V_vmux_QVV(wrap, Q6_Vw_vadd_VwVw(off, v_off_wrap), off);
|
||||
}
|
||||
}
|
||||
|
||||
// KH == 1: each output row is one patch, gathered from the staged [IC][Wb] window with a fixed offset
|
||||
// table shifted by s0 per row, then converted with a single copy.
|
||||
// scratch holds the offset table followed by the gathered row (both VTCM).
|
||||
#define IM2COL_GATHER_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN) \
|
||||
static void FNAME( \
|
||||
const struct htp_im2col_context * ictx, \
|
||||
void * restrict dstb_raw, const float * restrict srcb_c, \
|
||||
float * restrict scratch, \
|
||||
uint32_t nb, uint32_t c0, uint32_t ioh) { \
|
||||
DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \
|
||||
float * restrict srcb = (float *) srcb_c; \
|
||||
const int32_t s0 = ictx->octx->op_params[0], p0 = ictx->octx->op_params[2]; \
|
||||
const int32_t s1 = ictx->octx->op_params[1], p1 = ictx->octx->op_params[3]; \
|
||||
const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW, Wb = ictx->pe_wb; \
|
||||
const uint32_t patch_stride = ictx->patch_stride; \
|
||||
const uint32_t ng = ictx->pe_gather_nvec; \
|
||||
const int32_t win0 = (int32_t) c0 * s0 - p0; \
|
||||
const int32_t iih = (int32_t) ioh * s1 - p1; \
|
||||
\
|
||||
/* the DMA stage fills only the columns inside [0, IW): zero the rest of the window */ \
|
||||
const int32_t lo = win0 < 0 ? -win0 : 0; \
|
||||
const int32_t hi = MIN((int32_t) Wb, (int32_t) IW - win0); \
|
||||
if (iih < 0 || iih >= (int32_t) IH || hi <= lo) { \
|
||||
for (uint32_t iic = 0; iic < IC; iic++) { \
|
||||
hvx_splat_f32_u(srcb + (size_t) iic * Wb, 0, Wb); \
|
||||
} \
|
||||
} else if (lo > 0 || hi < (int32_t) Wb) { \
|
||||
for (uint32_t iic = 0; iic < IC; iic++) { \
|
||||
float * row = srcb + (size_t) iic * Wb; \
|
||||
if (lo > 0) { \
|
||||
hvx_splat_f32_u(row, 0, (uint32_t) lo); \
|
||||
} \
|
||||
if (hi < (int32_t) Wb) { \
|
||||
hvx_splat_f32_u(row + hi, 0, Wb - (uint32_t) hi); \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
const HVX_Vector * table = (const HVX_Vector *) scratch; \
|
||||
float * row = scratch + (size_t) ng * 32; \
|
||||
const uint32_t region = IC * Wb * sizeof(float); \
|
||||
\
|
||||
for (uint32_t j = 0; j < nb; j++) { \
|
||||
const uint32_t shift = j * (uint32_t) s0 * sizeof(float); \
|
||||
const size_t base = (size_t) ((const uint8_t *) srcb + shift); \
|
||||
const uint32_t mu = region - shift - 1; \
|
||||
for (uint32_t g = 0; g < ng; g++) { \
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (row + g * 32), base, mu, table[g]); \
|
||||
} \
|
||||
hvx_gather_sync(row + (ng - 1) * 32); \
|
||||
DST_CTYPE * out = dstb + (size_t) j * patch_stride; \
|
||||
COPY_FN((uint8_t *) out, (const uint8_t *) row, patch_stride); \
|
||||
} \
|
||||
}
|
||||
|
||||
IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f16, uint16_t, hvx_copy_f16_f32_uu)
|
||||
IM2COL_GATHER_COMPUTE_BODY(im2col_gather_compute_f32, float, hvx_copy_f32_uu)
|
||||
|
||||
static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) {
|
||||
(void) nth;
|
||||
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
|
||||
@@ -340,24 +234,24 @@ static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void *
|
||||
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
|
||||
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
|
||||
float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes);
|
||||
// work is split over output-col blocks, so a single output row (1D) still uses every thread
|
||||
const uint32_t blk_end_max = ictx->pe_nrows * nbpr;
|
||||
const uint32_t blk_start = ictx->pe_blocks_per_thread * ith;
|
||||
const uint32_t blk_end = MIN(blk_start + ictx->pe_blocks_per_thread, blk_end_max);
|
||||
if (blk_start >= blk_end) {
|
||||
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
|
||||
const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
|
||||
const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
|
||||
if (row_start >= row_end) {
|
||||
return;
|
||||
}
|
||||
const uint32_t nrows_local = row_end - row_start;
|
||||
const uint32_t total_blocks = nrows_local * nbpr;
|
||||
if (total_blocks == 0) {
|
||||
return;
|
||||
}
|
||||
const uint32_t total_blocks = blk_end - blk_start;
|
||||
struct im2col_cursor cur = {
|
||||
.r = ictx->thread_in[ith] * ictx->OH + ictx->thread_ioh[ith],
|
||||
.r = row_start,
|
||||
.in = ictx->thread_in[ith],
|
||||
.iod = 0,
|
||||
.ioh = ictx->thread_ioh[ith],
|
||||
.c0 = ictx->thread_c0[ith],
|
||||
.c0 = 0,
|
||||
};
|
||||
if (KH == 1) {
|
||||
im2col_gather_table_build(ictx, (HVX_Vector *) scratch);
|
||||
}
|
||||
struct im2col_cursor stage = cur;
|
||||
for (uint32_t i = 0; i < total_blocks && i < 2; i++) {
|
||||
dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0);
|
||||
@@ -413,37 +307,6 @@ static void im2col_patchembed_compute_f16(
|
||||
const uint32_t KH = ictx->KH, KW = ictx->KW;
|
||||
const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride;
|
||||
|
||||
if (KW == 16 && (KH % 4 == 0)) {
|
||||
const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
|
||||
for (uint32_t iow = 0; iow < OW; iow++) {
|
||||
uint16_t * dst_patch = dstb + (size_t) iow * patch_stride;
|
||||
for (uint32_t iic = 0; iic < IC; iic++) {
|
||||
for (uint32_t ikh = 0; ikh < KH; ikh += 4) {
|
||||
const int32_t iih0 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 0);
|
||||
const int32_t iih1 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 1);
|
||||
const int32_t iih2 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 2);
|
||||
const int32_t iih3 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 3);
|
||||
|
||||
const float * s0 = srcb + (size_t) (iic * KH + ikh + 0) * IW + (size_t) iow * 16;
|
||||
const float * s1 = srcb + (size_t) (iic * KH + ikh + 1) * IW + (size_t) iow * 16;
|
||||
const float * s2 = srcb + (size_t) (iic * KH + ikh + 2) * IW + (size_t) iow * 16;
|
||||
const float * s3 = srcb + (size_t) (iic * KH + ikh + 3) * IW + (size_t) iow * 16;
|
||||
|
||||
const HVX_Vector v0 = (iih0 >= 0 && iih0 < (int32_t) IH) ? *((const HVX_UVector *) s0) : Q6_V_vzero();
|
||||
const HVX_Vector v1 = (iih1 >= 0 && iih1 < (int32_t) IH) ? *((const HVX_UVector *) s1) : Q6_V_vzero();
|
||||
const HVX_Vector v2 = (iih2 >= 0 && iih2 < (int32_t) IH) ? *((const HVX_UVector *) s2) : Q6_V_vzero();
|
||||
const HVX_Vector v3 = (iih3 >= 0 && iih3 < (int32_t) IH) ? *((const HVX_UVector *) s3) : Q6_V_vzero();
|
||||
|
||||
const HVX_Vector w01 = Q6_V_vmux_QVV(q_lo, v0, Q6_V_vror_VR(v1, 64));
|
||||
const HVX_Vector w23 = Q6_V_vmux_QVV(q_lo, v2, Q6_V_vror_VR(v3, 64));
|
||||
|
||||
*((HVX_Vector *) (dst_patch + (size_t) (iic * KH + ikh) * 16)) = hvx_vec_f32_to_f16(w01, w23);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (uint32_t iow = 0; iow < OW; iow++) {
|
||||
uint16_t * dst_patch = dstb + (size_t) iow * patch_stride;
|
||||
for (uint32_t ikh = 0; ikh < KH; ikh++) {
|
||||
@@ -469,30 +332,6 @@ static void im2col_patchembed_compute_f32(
|
||||
const uint32_t KH = ictx->KH, KW = ictx->KW;
|
||||
const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride;
|
||||
|
||||
if (KW == 16 && (KH % 2 == 0)) {
|
||||
const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
|
||||
for (uint32_t iow = 0; iow < OW; iow++) {
|
||||
float * dst_patch = dstb + (size_t) iow * patch_stride;
|
||||
for (uint32_t iic = 0; iic < IC; iic++) {
|
||||
for (uint32_t ikh = 0; ikh < KH; ikh += 2) {
|
||||
const int32_t iih0 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 0);
|
||||
const int32_t iih1 = (int32_t) ioh * (int32_t) KH + (int32_t) (ikh + 1);
|
||||
|
||||
const float * s0 = srcb + (size_t) (iic * KH + ikh + 0) * IW + (size_t) iow * 16;
|
||||
const float * s1 = srcb + (size_t) (iic * KH + ikh + 1) * IW + (size_t) iow * 16;
|
||||
|
||||
const HVX_Vector v0 = (iih0 >= 0 && iih0 < (int32_t) IH) ? *((const HVX_UVector *) s0) : Q6_V_vzero();
|
||||
const HVX_Vector v1 = (iih1 >= 0 && iih1 < (int32_t) IH) ? *((const HVX_UVector *) s1) : Q6_V_vzero();
|
||||
|
||||
const HVX_Vector w01 = Q6_V_vmux_QVV(q_lo, v0, Q6_V_vror_VR(v1, 64));
|
||||
|
||||
*((HVX_Vector *) (dst_patch + (size_t) (iic * KH + ikh) * 16)) = w01;
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (uint32_t iow = 0; iow < OW; iow++) {
|
||||
float * dst_patch = dstb + (size_t) iow * patch_stride;
|
||||
for (uint32_t ikh = 0; ikh < KH; ikh++) {
|
||||
@@ -945,16 +784,12 @@ int op_im2col(struct htp_ops_context * octx) {
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
// the blocked path splits work over output-col blocks, so a single output row (1D) can use every thread
|
||||
const bool blocked = kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA;
|
||||
const uint32_t nwork = blocked ? nrows * kparams->nbpr : nrows;
|
||||
if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nwork))) {
|
||||
if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
|
||||
struct htp_im2col_vtcm_layout layout;
|
||||
htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads,
|
||||
htp_im2col_scratch_bytes(kparams->kernel_type, kparams->KH, kparams->patch_stride));
|
||||
htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
|
||||
|
||||
struct htp_im2col_context ictx = { 0 };
|
||||
uint8_t * const base = octx->ctx->vtcm_base;
|
||||
@@ -964,7 +799,6 @@ int op_im2col(struct htp_ops_context * octx) {
|
||||
ictx.pe_row_base = row_base;
|
||||
ictx.pe_nrows = nrows;
|
||||
ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
|
||||
ictx.pe_blocks_per_thread = fastdiv(nwork + pth - 1, &octx->n_threads_div);
|
||||
ictx.pe_src_row_bytes = kparams->src_row_bytes;
|
||||
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
|
||||
ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
|
||||
@@ -986,22 +820,12 @@ int op_im2col(struct htp_ops_context * octx) {
|
||||
ictx.OH = kparams->OH;
|
||||
ictx.OW = kparams->OW;
|
||||
ictx.patch_stride = kparams->patch_stride;
|
||||
ictx.pe_gather_nvec = hex_round_up(kparams->patch_stride, 32) / 32;
|
||||
|
||||
// a thread starts at a row, or at an output-col block (possibly mid-row) on the blocked path
|
||||
for (uint32_t ith = 0; ith < pth; ith++) {
|
||||
uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
|
||||
uint32_t c0 = 0;
|
||||
if (blocked) {
|
||||
const uint32_t blk = ictx.pe_blocks_per_thread * ith;
|
||||
const uint32_t q = fastdiv(blk, &kparams->div_nbpr);
|
||||
r = row_base + q;
|
||||
c0 = (blk - q * kparams->nbpr) * kparams->owb;
|
||||
}
|
||||
const uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
|
||||
const uint32_t in = fastdiv(r, &kparams->div_OH);
|
||||
ictx.thread_ioh[ith] = r - in * kparams->OH;
|
||||
ictx.thread_in[ith] = in;
|
||||
ictx.thread_c0[ith] = c0;
|
||||
}
|
||||
|
||||
ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);
|
||||
@@ -1011,14 +835,9 @@ int op_im2col(struct htp_ops_context * octx) {
|
||||
: (void *) im2col_patchembed_compute_f32;
|
||||
work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth);
|
||||
return HTP_STATUS_OK;
|
||||
} else if (blocked) {
|
||||
if (kparams->KH == 1) {
|
||||
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_gather_compute_f16
|
||||
: (void *) im2col_gather_compute_f32;
|
||||
} else {
|
||||
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16
|
||||
: (void *) im2col_blocked_compute_f32;
|
||||
}
|
||||
} else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) {
|
||||
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16
|
||||
: (void *) im2col_blocked_compute_f32;
|
||||
work_queue_run(octx->ctx->work_queue, im2col_blocked_dma_thread, &ictx, pth);
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
@@ -1054,7 +873,7 @@ int op_im2col_3d(struct htp_ops_context * octx) {
|
||||
}
|
||||
|
||||
struct htp_im2col_vtcm_layout layout;
|
||||
htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads, 256);
|
||||
htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
|
||||
|
||||
struct htp_im2col_context ictx = { 0 };
|
||||
uint8_t * const base = octx->ctx->vtcm_base;
|
||||
|
||||
@@ -20,15 +20,13 @@ struct htp_im2col_vtcm_layout {
|
||||
uint32_t dst_bytes_per_thread;
|
||||
};
|
||||
|
||||
// scratch_bytes: per-thread VTCM after the two src buffers (at least 256)
|
||||
static inline void htp_im2col_vtcm_layout_build(
|
||||
struct htp_im2col_vtcm_layout * L,
|
||||
uint32_t src_row_bytes,
|
||||
uint32_t dst_row_bytes,
|
||||
uint32_t n_threads,
|
||||
uint32_t scratch_bytes) {
|
||||
uint32_t n_threads) {
|
||||
|
||||
L->src_bytes_per_thread = 2 * src_row_bytes + (scratch_bytes > 256 ? scratch_bytes : 256);
|
||||
L->src_bytes_per_thread = 2 * src_row_bytes + 256;
|
||||
L->dst_bytes_per_thread = 2 * dst_row_bytes;
|
||||
|
||||
size_t off = 0;
|
||||
@@ -37,15 +35,6 @@ static inline void htp_im2col_vtcm_layout_build(
|
||||
L->total_bytes = off;
|
||||
}
|
||||
|
||||
// per-thread scratch: the KH == 1 blocked path keeps its offset table and one gathered row there,
|
||||
// both padded to whole vectors
|
||||
static inline uint32_t htp_im2col_scratch_bytes(uint32_t kernel_type, uint32_t KH, uint32_t patch_stride) {
|
||||
if (kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA && KH == 1) {
|
||||
return 2 * hex_round_up(patch_stride, 32) * (uint32_t) sizeof(float);
|
||||
}
|
||||
return 256;
|
||||
}
|
||||
|
||||
struct htp_im2col_kernel_params {
|
||||
uint8_t kernel_type;
|
||||
uint8_t n_threads;
|
||||
@@ -69,7 +58,6 @@ struct htp_im2col_kernel_params {
|
||||
|
||||
struct fastdiv_values div_OH;
|
||||
struct fastdiv_values div_OD;
|
||||
struct fastdiv_values div_nbpr;
|
||||
};
|
||||
|
||||
#if defined(__cplusplus)
|
||||
|
||||
@@ -677,7 +677,7 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
|
||||
dma_ptr_curr += size_dma_q;
|
||||
}
|
||||
|
||||
ctx->ddr_spad_size = HTP_DDR_SPAD_SIZE;
|
||||
ctx->ddr_spad_size = 512 * 1024; // 512 KB
|
||||
ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size);
|
||||
|
||||
void * wq_ptr = (void *) ((uintptr_t) block + offset_wq);
|
||||
@@ -1025,12 +1025,6 @@ static int execute_op(struct htp_ops_context * octx) {
|
||||
case HTP_OP_IM2COL_3D:
|
||||
return op_im2col_3d(octx);
|
||||
|
||||
case HTP_OP_CONV_2D:
|
||||
return op_conv_2d(octx);
|
||||
|
||||
case HTP_OP_CONV_2D_DW:
|
||||
return op_conv_2d_dw(octx);
|
||||
|
||||
case HTP_OP_ROLL:
|
||||
return op_roll(octx);
|
||||
|
||||
@@ -1049,9 +1043,6 @@ static int execute_op(struct htp_ops_context * octx) {
|
||||
case HTP_OP_TRI:
|
||||
return op_unary(octx);
|
||||
|
||||
case HTP_OP_UPSCALE:
|
||||
return op_upscale(octx);
|
||||
|
||||
case HTP_OP_INVALID:
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -173,12 +173,11 @@ static inline void hvx_ssm_conv_decode_4(const float * x, const float * w, float
|
||||
// the lanes past the channel tail.
|
||||
static inline void hvx_ssm_conv_transpose_block(const float * raw_block,
|
||||
float * T,
|
||||
uint32_t raw_stride_bytes,
|
||||
uint32_t ncs,
|
||||
uint32_t cb_n,
|
||||
HVX_Vector vv) {
|
||||
const size_t base = (size_t) raw_block;
|
||||
const uint32_t mu = cb_n * raw_stride_bytes - 1;
|
||||
const uint32_t mu = cb_n * ncs * sizeof(float) - 1;
|
||||
|
||||
for (uint32_t t = 0; t < ncs; ++t) {
|
||||
Q6_vgather_ARMVw((HVX_Vector *) (T + (size_t) t * VLEN_FP32), base + t * sizeof(float), mu, vv);
|
||||
@@ -394,8 +393,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
|
||||
|
||||
// src0 spad holds two raw tiles (fetch of tile n+1 overlaps compute of tile n) plus
|
||||
// the transposed block. dst spad holds two tiles so a writeback can stay in flight.
|
||||
const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float);
|
||||
const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * raw_stride_bytes, 128);
|
||||
const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * ncs * sizeof(float), 128);
|
||||
const size_t dst_tile_bytes = hex_round_up(d_inner_tile * n_t * sizeof(float), 128);
|
||||
|
||||
float * src0_tile_raw[2] = { (float *) src0_spad_base, (float *) (src0_spad_base + src0_tile_raw_bytes) };
|
||||
@@ -421,7 +419,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
|
||||
(ir0 + tile_off_fetch) * src0_stride_inner_bytes; \
|
||||
dma_queue_push(dma_q, \
|
||||
dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \
|
||||
raw_stride_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \
|
||||
row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \
|
||||
tile_off_fetch += d_inner_tile; \
|
||||
if (tile_off_fetch >= d_inner_per_thread) { \
|
||||
tile_off_fetch = 0; \
|
||||
@@ -448,7 +446,11 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
|
||||
const uint32_t C_TILE = VLEN_FP32;
|
||||
|
||||
// gather offsets: lane r reads channel r of the raw tile block
|
||||
const HVX_Vector vv = hvx_vec_gather_offsets_w(raw_stride_bytes);
|
||||
uint32_t __attribute__((aligned(VLEN))) gather_off[VLEN_FP32];
|
||||
for (uint32_t r = 0; r < VLEN_FP32; ++r) {
|
||||
gather_off[r] = r * ncs * sizeof(float);
|
||||
}
|
||||
const HVX_Vector vv = *(const HVX_Vector *) gather_off;
|
||||
|
||||
uint32_t i3 = 0;
|
||||
uint32_t tile_off = 0;
|
||||
@@ -472,8 +474,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
|
||||
const uint32_t cb_n = MIN(C_TILE, tile_n - cb);
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off);
|
||||
const float * raw_ch = (const float *) ((const uint8_t *) raw + (size_t) cb * raw_stride_bytes);
|
||||
hvx_ssm_conv_transpose_block(raw_ch, src0_T, raw_stride_bytes, ncs, cb_n, vv);
|
||||
hvx_ssm_conv_transpose_block(raw + (size_t) cb * ncs, src0_T, ncs, cb_n, vv);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off);
|
||||
|
||||
const float * restrict wp = src1_T + tile_off + cb;
|
||||
|
||||
@@ -54,12 +54,11 @@ static inline void htp_ssm_conv_vtcm_layout_build(
|
||||
L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes;
|
||||
L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128);
|
||||
} else {
|
||||
const uint32_t ncs = d_conv - 1 + n_t;
|
||||
const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float);
|
||||
const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128);
|
||||
const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * raw_stride_bytes, 128);
|
||||
L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T;
|
||||
L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);
|
||||
const uint32_t ncs = d_conv - 1 + n_t;
|
||||
const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128);
|
||||
const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * ncs * sizeof(float), 128);
|
||||
L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T;
|
||||
L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);
|
||||
}
|
||||
|
||||
L->src0_bytes = L->src0_bytes_per_thread * n_threads;
|
||||
|
||||
@@ -1,958 +0,0 @@
|
||||
#pragma clang diagnostic ignored "-Wunused-variable"
|
||||
|
||||
#include <float.h>
|
||||
#include <math.h>
|
||||
#include <HAP_farf.h>
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "dma-queue.h"
|
||||
#include "hex-profile.h"
|
||||
#include "htp-ctx.h"
|
||||
#include "htp-ops.h"
|
||||
#include "htp-tensor.h"
|
||||
#include "hvx-types.h"
|
||||
#include "hvx-base.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "upscale-ops.h"
|
||||
|
||||
static inline int32_t clamp_i32(int32_t v, int32_t lo, int32_t hi) {
|
||||
return v < lo ? lo : (v > hi ? hi : v);
|
||||
}
|
||||
|
||||
static inline float clamp_f32(float v, float lo, float hi) {
|
||||
return v < lo ? lo : (v > hi ? hi : v);
|
||||
}
|
||||
|
||||
static inline float bicubic_weight1(float x) {
|
||||
const float a = -0.75f;
|
||||
return ((a + 2.0f) * x - (a + 3.0f)) * x * x + 1.0f;
|
||||
}
|
||||
|
||||
static inline float bicubic_weight2(float x) {
|
||||
const float a = -0.75f;
|
||||
return ((a * x - 5.0f * a) * x + 8.0f * a) * x - 4.0f * a;
|
||||
}
|
||||
|
||||
static inline void bicubic_weights(float x, float w[4]) {
|
||||
w[0] = bicubic_weight2(x + 1.0f);
|
||||
w[1] = bicubic_weight1(x);
|
||||
w[2] = bicubic_weight1(1.0f - x);
|
||||
w[3] = bicubic_weight2(2.0f - x);
|
||||
}
|
||||
|
||||
static void upscale_init_luts_nearest(
|
||||
uint32_t * lut_x,
|
||||
uint32_t * lut_y,
|
||||
const struct htp_tensor * src0,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_upscale_kernel_params * p
|
||||
) {
|
||||
const uint32_t src_x = src0->ne[0];
|
||||
const uint32_t src_y = src0->ne[1];
|
||||
const uint32_t dst_x = dst->ne[0];
|
||||
const uint32_t dst_y = dst->ne[1];
|
||||
const uint32_t dst_x_aligned = (dst_x + VLEN_FP32 - 1) & ~(VLEN_FP32 - 1);
|
||||
|
||||
for (uint32_t ox = 0; ox < dst_x_aligned; ++ox) {
|
||||
if (ox < dst_x) {
|
||||
uint32_t x0 = (uint32_t) ((float) ox * p->inv_sf0);
|
||||
lut_x[ox] = MIN(x0, src_x - 1) * sizeof(float);
|
||||
} else {
|
||||
lut_x[ox] = 0;
|
||||
}
|
||||
}
|
||||
for (uint32_t oy = 0; oy < dst_y; ++oy) {
|
||||
uint32_t y0 = (uint32_t) ((float) oy * p->inv_sf1);
|
||||
lut_y[oy] = MIN(y0, src_y - 1);
|
||||
}
|
||||
}
|
||||
|
||||
static void upscale_init_luts_bilinear(
|
||||
struct upscale_bilinear_x_chunk * lut_x,
|
||||
struct upscale_bilinear_y * lut_y,
|
||||
const struct htp_tensor * src0,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_upscale_kernel_params * p
|
||||
) {
|
||||
const uint32_t src_x = src0->ne[0];
|
||||
const uint32_t src_y = src0->ne[1];
|
||||
const uint32_t dst_x = dst->ne[0];
|
||||
const uint32_t dst_y = dst->ne[1];
|
||||
const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32;
|
||||
|
||||
for (uint32_t c = 0; c < n_chunks; ++c) {
|
||||
struct upscale_bilinear_x_chunk * chunk = &lut_x[c];
|
||||
const uint32_t ox_start = c * VLEN_FP32;
|
||||
for (uint32_t l = 0; l < VLEN_FP32; ++l) {
|
||||
const uint32_t ox = ox_start + l;
|
||||
if (ox < dst_x) {
|
||||
float xf = ((float) ox + p->pixel_offset) * p->inv_sf0 - p->pixel_offset;
|
||||
int32_t x0 = (int32_t) floorf(xf);
|
||||
int32_t x1 = x0 + 1;
|
||||
x0 = clamp_i32(x0, 0, (int32_t) src_x - 1);
|
||||
x1 = clamp_i32(x1, 0, (int32_t) src_x - 1);
|
||||
float dx = clamp_f32(xf - (float) x0, 0.0f, 1.0f);
|
||||
|
||||
chunk->off0[l] = x0 * (int32_t) sizeof(float);
|
||||
chunk->off1[l] = x1 * (int32_t) sizeof(float);
|
||||
chunk->w0[l] = 1.0f - dx;
|
||||
chunk->dx[l] = dx;
|
||||
} else {
|
||||
chunk->off0[l] = 0;
|
||||
chunk->off1[l] = 0;
|
||||
chunk->w0[l] = 0.0f;
|
||||
chunk->dx[l] = 0.0f;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (uint32_t oy = 0; oy < dst_y; ++oy) {
|
||||
float yf = ((float) oy + p->pixel_offset) * p->inv_sf1 - p->pixel_offset;
|
||||
int32_t y0 = (int32_t) floorf(yf);
|
||||
int32_t y1 = y0 + 1;
|
||||
y0 = clamp_i32(y0, 0, (int32_t) src_y - 1);
|
||||
y1 = clamp_i32(y1, 0, (int32_t) src_y - 1);
|
||||
float dy = clamp_f32(yf - (float) y0, 0.0f, 1.0f);
|
||||
|
||||
lut_y[oy].y0 = (uint32_t) y0;
|
||||
lut_y[oy].y1 = (uint32_t) y1;
|
||||
lut_y[oy].wy1 = dy;
|
||||
lut_y[oy].wy0 = 1.0f - dy;
|
||||
}
|
||||
}
|
||||
|
||||
static void upscale_init_luts_bicubic(
|
||||
struct upscale_bicubic_x * lut_x,
|
||||
struct upscale_bicubic_y * lut_y,
|
||||
const struct htp_tensor * src0,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_upscale_kernel_params * p
|
||||
) {
|
||||
const uint32_t src_x = src0->ne[0];
|
||||
const uint32_t src_y = src0->ne[1];
|
||||
const uint32_t dst_x = dst->ne[0];
|
||||
const uint32_t dst_y = dst->ne[1];
|
||||
|
||||
for (uint32_t ox = 0; ox < dst_x; ++ox) {
|
||||
float xf = ((float) ox + p->pixel_offset) * p->inv_sf0 - p->pixel_offset;
|
||||
int32_t x0 = (int32_t) floorf(xf);
|
||||
float dx = xf - (float) x0;
|
||||
bicubic_weights(dx, lut_x[ox].w);
|
||||
|
||||
lut_x[ox].x[0] = clamp_i32(x0 - 1, 0, (int32_t) src_x - 1);
|
||||
lut_x[ox].x[1] = clamp_i32(x0 + 0, 0, (int32_t) src_x - 1);
|
||||
lut_x[ox].x[2] = clamp_i32(x0 + 1, 0, (int32_t) src_x - 1);
|
||||
lut_x[ox].x[3] = clamp_i32(x0 + 2, 0, (int32_t) src_x - 1);
|
||||
}
|
||||
for (uint32_t oy = 0; oy < dst_y; ++oy) {
|
||||
float yf = ((float) oy + p->pixel_offset) * p->inv_sf1 - p->pixel_offset;
|
||||
int32_t y0 = (int32_t) floorf(yf);
|
||||
float dy = yf - (float) y0;
|
||||
bicubic_weights(dy, lut_y[oy].w);
|
||||
|
||||
lut_y[oy].y[0] = clamp_i32(y0 - 1, 0, (int32_t) src_y - 1);
|
||||
lut_y[oy].y[1] = clamp_i32(y0 + 0, 0, (int32_t) src_y - 1);
|
||||
lut_y[oy].y[2] = clamp_i32(y0 + 1, 0, (int32_t) src_y - 1);
|
||||
lut_y[oy].y[3] = clamp_i32(y0 + 2, 0, (int32_t) src_y - 1);
|
||||
}
|
||||
}
|
||||
|
||||
static void upscale_init_luts_antialias(
|
||||
struct upscale_antialias_x * lut_x,
|
||||
struct upscale_antialias_y * lut_y,
|
||||
const struct htp_tensor * src0,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_upscale_kernel_params * p
|
||||
) {
|
||||
const uint32_t src_x = src0->ne[0];
|
||||
const uint32_t src_y = src0->ne[1];
|
||||
const uint32_t dst_x = dst->ne[0];
|
||||
const uint32_t dst_y = dst->ne[1];
|
||||
|
||||
const float filter_radius0 = p->filter_radius0;
|
||||
const float filter_radius1 = p->filter_radius1;
|
||||
|
||||
for (uint32_t ox = 0; ox < dst_x; ++ox) {
|
||||
const float xf = ((float) ox + p->pixel_offset) * p->inv_sf0;
|
||||
int32_t x_min = (int32_t) MAX(xf - filter_radius0 + p->pixel_offset, 0.0f);
|
||||
int32_t x_max = (int32_t) MIN(xf + filter_radius0 + p->pixel_offset, (float) src_x);
|
||||
lut_x[ox].x_min = clamp_i32(x_min, 0, (int32_t) src_x);
|
||||
lut_x[ox].x_max = clamp_i32(x_max, 0, (int32_t) src_x);
|
||||
lut_x[ox].xf = xf;
|
||||
}
|
||||
for (uint32_t oy = 0; oy < dst_y; ++oy) {
|
||||
const float yf = ((float) oy + p->pixel_offset) * p->inv_sf1;
|
||||
int32_t y_min = (int32_t) MAX(yf - filter_radius1 + p->pixel_offset, 0.0f);
|
||||
int32_t y_max = (int32_t) MIN(yf + filter_radius1 + p->pixel_offset, (float) src_y);
|
||||
lut_y[oy].y_min = clamp_i32(y_min, 0, (int32_t) src_y);
|
||||
lut_y[oy].y_max = clamp_i32(y_max, 0, (int32_t) src_y);
|
||||
lut_y[oy].yf = yf;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void get_src_y_window(
|
||||
const struct htp_tensor * src0,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_upscale_kernel_params * p,
|
||||
const void * lut_y,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t * out_src_y_start,
|
||||
uint32_t * out_src_y_count
|
||||
) {
|
||||
const uint32_t src_y = src0->ne[1];
|
||||
const uint32_t dst_y = dst->ne[1];
|
||||
|
||||
if (dst_y_count == dst_y) {
|
||||
*out_src_y_start = 0;
|
||||
*out_src_y_count = src_y;
|
||||
return;
|
||||
}
|
||||
|
||||
const uint32_t last_oy = dst_y_start + dst_y_count - 1;
|
||||
|
||||
if (p->mode == HTP_UPSCALE_MODE_NEAREST) {
|
||||
const uint32_t * ly = (const uint32_t *) lut_y;
|
||||
const uint32_t y0 = ly[dst_y_start];
|
||||
const uint32_t y1 = ly[last_oy];
|
||||
*out_src_y_start = y0;
|
||||
*out_src_y_count = y1 - y0 + 1;
|
||||
} else if (p->mode == HTP_UPSCALE_MODE_BICUBIC) {
|
||||
const struct upscale_bicubic_y * ly = (const struct upscale_bicubic_y *) lut_y;
|
||||
const int32_t y0 = ly[dst_y_start].y[0];
|
||||
const int32_t y1 = ly[last_oy].y[3];
|
||||
*out_src_y_start = (uint32_t) y0;
|
||||
*out_src_y_count = (uint32_t) (y1 - y0 + 1);
|
||||
} else if (p->antialias) {
|
||||
const struct upscale_antialias_y * ly = (const struct upscale_antialias_y *) lut_y;
|
||||
const int32_t y0 = ly[dst_y_start].y_min;
|
||||
const int32_t y1 = ly[last_oy].y_max;
|
||||
*out_src_y_start = (uint32_t) y0;
|
||||
*out_src_y_count = (uint32_t) (y1 - y0);
|
||||
} else {
|
||||
const struct upscale_bilinear_y * ly = (const struct upscale_bilinear_y *) lut_y;
|
||||
const uint32_t y0 = ly[dst_y_start].y0;
|
||||
const uint32_t y1 = ly[last_oy].y1;
|
||||
*out_src_y_start = y0;
|
||||
*out_src_y_count = y1 - y0 + 1;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void resolve_dst_slice(
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_upscale_kernel_params * p,
|
||||
uint32_t chunk_idx,
|
||||
dma_addr_t * out_slice_dst_off,
|
||||
uint32_t * out_dst_ys,
|
||||
uint32_t * out_dst_yc
|
||||
) {
|
||||
const uint32_t s_idx = fastdiv(chunk_idx, &p->div_chunks_per_slice);
|
||||
const uint32_t c_idx = chunk_idx - s_idx * p->chunks_per_slice;
|
||||
const uint32_t d_ys = c_idx * p->dst_nrows_per_chunk;
|
||||
const uint32_t d_yc = MIN(p->dst_nrows_per_chunk, dst->ne[1] - d_ys);
|
||||
const uint32_t i3 = fastdiv(s_idx, &p->div_dst_z);
|
||||
const uint32_t i2 = s_idx - i3 * dst->ne[2];
|
||||
|
||||
*out_slice_dst_off = (dma_addr_t) i2 * dst->nb[2] + (dma_addr_t) i3 * dst->nb[3];
|
||||
*out_dst_ys = d_ys;
|
||||
*out_dst_yc = d_yc;
|
||||
}
|
||||
|
||||
static inline void resolve_src_slice(
|
||||
const struct htp_tensor * src0,
|
||||
const struct htp_tensor * dst,
|
||||
const struct htp_upscale_kernel_params * p,
|
||||
const void * lut_y,
|
||||
uint32_t chunk_idx,
|
||||
dma_addr_t * out_slice_src_off,
|
||||
uint32_t * out_src_ys,
|
||||
uint32_t * out_src_yc
|
||||
) {
|
||||
const uint32_t s_idx = fastdiv(chunk_idx, &p->div_chunks_per_slice);
|
||||
const uint32_t c_idx = chunk_idx - s_idx * p->chunks_per_slice;
|
||||
const uint32_t d_ys = c_idx * p->dst_nrows_per_chunk;
|
||||
const uint32_t d_yc = MIN(p->dst_nrows_per_chunk, dst->ne[1] - d_ys);
|
||||
uint32_t s_ys, s_yc;
|
||||
get_src_y_window(src0, dst, p, lut_y, d_ys, d_yc, &s_ys, &s_yc);
|
||||
|
||||
const uint32_t i3 = fastdiv(s_idx, &p->div_dst_z);
|
||||
const uint32_t i2 = s_idx - i3 * dst->ne[2];
|
||||
const uint32_t i03 = (uint32_t) ((float) i3 * p->inv_sf3);
|
||||
const uint32_t i02 = (uint32_t) ((float) i2 * p->inv_sf2);
|
||||
|
||||
*out_slice_src_off = (dma_addr_t) i02 * src0->nb[2] + (dma_addr_t) i03 * src0->nb[3];
|
||||
*out_src_ys = s_ys;
|
||||
*out_src_yc = s_yc;
|
||||
}
|
||||
|
||||
struct upscale_context;
|
||||
|
||||
typedef void (*upscale_chunk_fn_t)(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
);
|
||||
|
||||
struct upscale_context {
|
||||
struct htp_ops_context * octx;
|
||||
const struct htp_upscale_kernel_params * kparams;
|
||||
struct htp_upscale_vtcm_layout layout;
|
||||
upscale_chunk_fn_t compute_chunk;
|
||||
const void * lut_x;
|
||||
const void * lut_y;
|
||||
uint32_t chunk_start;
|
||||
uint32_t chunk_count;
|
||||
uint32_t chunks_per_thread;
|
||||
};
|
||||
|
||||
static void compute_nearest_chunk(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
) {
|
||||
const uint32_t src_x = ctx->octx->src[0]->ne[0];
|
||||
const uint32_t dst_x = ctx->octx->dst->ne[0];
|
||||
const uint32_t * lut_x = (const uint32_t *) ctx->lut_x;
|
||||
const uint32_t * lut_y = (const uint32_t *) ctx->lut_y;
|
||||
const uint32_t region_size = hex_round_up(src_x * sizeof(float), 128);
|
||||
|
||||
for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
|
||||
const uint32_t y0 = lut_y[oy];
|
||||
const float * r0 = srcb + (y0 - src_y_start) * src_x;
|
||||
float * dst_row = dstb + (oy - dst_y_start) * dst_x;
|
||||
|
||||
for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
|
||||
*(HVX_Vector *) (R_buf + ix) = hvx_vmemu(r0 + ix);
|
||||
}
|
||||
|
||||
for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) {
|
||||
const uint32_t rem = dst_x - ox;
|
||||
const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
|
||||
const HVX_Vector voff = hvx_vmem(lut_x + ox);
|
||||
Q6_vgather_ARMVw((HVX_Vector *) gather_tmp, (size_t) R_buf, region_size, voff);
|
||||
hvx_gather_sync(gather_tmp);
|
||||
hvx_vec_store_u(dst_row + ox, nbytes, hvx_vmem(gather_tmp));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void compute_nearest_2x_chunk(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
) {
|
||||
(void) R_buf;
|
||||
(void) gather_tmp;
|
||||
const uint32_t src_x = ctx->octx->src[0]->ne[0];
|
||||
const uint32_t dst_x = ctx->octx->dst->ne[0];
|
||||
|
||||
for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
|
||||
const uint32_t sy = oy >> 1;
|
||||
const float * r0 = srcb + (sy - src_y_start) * src_x;
|
||||
float * dst_row = dstb + (oy - dst_y_start) * dst_x;
|
||||
|
||||
for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
|
||||
const HVX_Vector vin = hvx_vmemu(r0 + ix);
|
||||
|
||||
const HVX_VectorPair p = Q6_W_vshuff_VVR(vin, vin, -4);
|
||||
const HVX_Vector v_lo = Q6_V_lo_W(p);
|
||||
const HVX_Vector v_hi = Q6_V_hi_W(p);
|
||||
|
||||
const uint32_t ox0 = 2 * ix;
|
||||
const uint32_t rem_out0 = (dst_x > ox0) ? (dst_x - ox0) : 0;
|
||||
const uint32_t nbytes_out0 = (rem_out0 < VLEN_FP32) ? (rem_out0 * sizeof(float)) : VLEN;
|
||||
hvx_vec_store_u(dst_row + ox0, nbytes_out0, v_lo);
|
||||
|
||||
const uint32_t ox1 = ox0 + VLEN_FP32;
|
||||
if (dst_x > ox1) {
|
||||
const uint32_t rem_out1 = dst_x - ox1;
|
||||
const uint32_t nbytes_out1 = (rem_out1 < VLEN_FP32) ? (rem_out1 * sizeof(float)) : VLEN;
|
||||
hvx_vec_store_u(dst_row + ox1, nbytes_out1, v_hi);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void compute_nearest_4x_chunk(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
) {
|
||||
(void) R_buf;
|
||||
(void) gather_tmp;
|
||||
const uint32_t src_x = ctx->octx->src[0]->ne[0];
|
||||
const uint32_t dst_x = ctx->octx->dst->ne[0];
|
||||
|
||||
for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
|
||||
const uint32_t sy = oy >> 2;
|
||||
const float * r0 = srcb + (sy - src_y_start) * src_x;
|
||||
float * dst_row = dstb + (oy - dst_y_start) * dst_x;
|
||||
|
||||
for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
|
||||
const HVX_Vector vin = hvx_vmemu(r0 + ix);
|
||||
|
||||
const HVX_VectorPair p = Q6_W_vshuff_VVR(vin, vin, -4);
|
||||
const HVX_Vector v_lo = Q6_V_lo_W(p);
|
||||
const HVX_Vector v_hi = Q6_V_hi_W(p);
|
||||
|
||||
const HVX_VectorPair p0 = Q6_W_vshuff_VVR(v_lo, v_lo, -4);
|
||||
const HVX_VectorPair p1 = Q6_W_vshuff_VVR(v_hi, v_hi, -4);
|
||||
|
||||
const HVX_Vector out0 = Q6_V_lo_W(p0);
|
||||
const HVX_Vector out1 = Q6_V_hi_W(p0);
|
||||
const HVX_Vector out2 = Q6_V_lo_W(p1);
|
||||
const HVX_Vector out3 = Q6_V_hi_W(p1);
|
||||
|
||||
const uint32_t ox = 4 * ix;
|
||||
if (dst_x > ox) {
|
||||
const uint32_t rem = dst_x - ox;
|
||||
hvx_vec_store_u(dst_row + ox, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out0);
|
||||
}
|
||||
if (dst_x > ox + VLEN_FP32) {
|
||||
const uint32_t rem = dst_x - (ox + VLEN_FP32);
|
||||
hvx_vec_store_u(dst_row + ox + VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out1);
|
||||
}
|
||||
if (dst_x > ox + 2 * VLEN_FP32) {
|
||||
const uint32_t rem = dst_x - (ox + 2 * VLEN_FP32);
|
||||
hvx_vec_store_u(dst_row + ox + 2 * VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out2);
|
||||
}
|
||||
if (dst_x > ox + 3 * VLEN_FP32) {
|
||||
const uint32_t rem = dst_x - (ox + 3 * VLEN_FP32);
|
||||
hvx_vec_store_u(dst_row + ox + 3 * VLEN_FP32, (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN, out3);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void compute_bilinear_chunk_32(
|
||||
float * restrict dst_chunk,
|
||||
const float * restrict R_buf,
|
||||
float * restrict tmp_a,
|
||||
float * restrict tmp_b,
|
||||
uint32_t region_size,
|
||||
const struct upscale_bilinear_x_chunk * restrict chunk,
|
||||
uint32_t nbytes
|
||||
) {
|
||||
const HVX_Vector voff0 = hvx_vmem(chunk->off0);
|
||||
const HVX_Vector voff1 = hvx_vmem(chunk->off1);
|
||||
const HVX_Vector vw0 = hvx_vmem(chunk->w0);
|
||||
const HVX_Vector vdx = hvx_vmem(chunk->dx);
|
||||
|
||||
Q6_vgather_ARMVw((HVX_Vector *) tmp_a, (size_t) R_buf, region_size, voff0);
|
||||
Q6_vgather_ARMVw((HVX_Vector *) tmp_b, (size_t) R_buf, region_size, voff1);
|
||||
hvx_gather_sync(tmp_b);
|
||||
|
||||
const HVX_Vector va = hvx_vmem(tmp_a);
|
||||
const HVX_Vector vb = hvx_vmem(tmp_b);
|
||||
|
||||
const HVX_Vector val = hvx_vec_add_f32_f32(
|
||||
hvx_vec_mul_f32_f32(vw0, va),
|
||||
hvx_vec_mul_f32_f32(vdx, vb)
|
||||
);
|
||||
|
||||
hvx_vec_store_u(dst_chunk, nbytes, val);
|
||||
}
|
||||
|
||||
static void compute_bilinear_chunk(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
) {
|
||||
const uint32_t src_x = ctx->octx->src[0]->ne[0];
|
||||
const uint32_t dst_x = ctx->octx->dst->ne[0];
|
||||
const struct upscale_bilinear_x_chunk * lut_x = (const struct upscale_bilinear_x_chunk *) ctx->lut_x;
|
||||
const struct upscale_bilinear_y * lut_y = (const struct upscale_bilinear_y *) ctx->lut_y;
|
||||
const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32;
|
||||
const uint32_t region_size = hex_round_up(src_x * sizeof(float), 128);
|
||||
float * tmp_a = gather_tmp;
|
||||
float * tmp_b = gather_tmp + VLEN_FP32;
|
||||
|
||||
for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
|
||||
const struct upscale_bilinear_y * ey = &lut_y[oy];
|
||||
const float * r0 = srcb + (ey->y0 - src_y_start) * src_x;
|
||||
const float * r1 = srcb + (ey->y1 - src_y_start) * src_x;
|
||||
float * dst_row = dstb + (oy - dst_y_start) * dst_x;
|
||||
|
||||
if (ey->y0 == ey->y1) {
|
||||
for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
|
||||
*(HVX_Vector *) (R_buf + ix) = hvx_vmemu(r0 + ix);
|
||||
}
|
||||
} else {
|
||||
const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->wy0);
|
||||
const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->wy1);
|
||||
for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
|
||||
const HVX_Vector vr0 = hvx_vmemu(r0 + ix);
|
||||
const HVX_Vector vr1 = hvx_vmemu(r1 + ix);
|
||||
*(HVX_Vector *) (R_buf + ix) = hvx_vec_add_f32_f32(
|
||||
hvx_vec_mul_f32_f32(vwy0, vr0),
|
||||
hvx_vec_mul_f32_f32(vwy1, vr1)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (uint32_t c = 0; c < n_chunks; ++c) {
|
||||
const uint32_t ox = c * VLEN_FP32;
|
||||
const uint32_t rem = dst_x - ox;
|
||||
const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
|
||||
compute_bilinear_chunk_32(dst_row + ox, R_buf, tmp_a, tmp_b, region_size, &lut_x[c], nbytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void compute_bilinear_2x_chunk(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
) {
|
||||
(void) R_buf;
|
||||
(void) gather_tmp;
|
||||
const uint32_t src_x = ctx->octx->src[0]->ne[0];
|
||||
const uint32_t dst_x = ctx->octx->dst->ne[0];
|
||||
const struct upscale_bilinear_y * lut_y = (const struct upscale_bilinear_y *) ctx->lut_y;
|
||||
const HVX_Vector c_075 = hvx_vec_splat_f32(0.75f);
|
||||
const HVX_Vector c_025 = hvx_vec_splat_f32(0.25f);
|
||||
|
||||
for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
|
||||
const struct upscale_bilinear_y * ey = &lut_y[oy];
|
||||
const float * r0 = srcb + (ey->y0 - src_y_start) * src_x;
|
||||
const float * r1 = srcb + (ey->y1 - src_y_start) * src_x;
|
||||
float * dst_row = dstb + (oy - dst_y_start) * dst_x;
|
||||
|
||||
const bool same_row = (ey->y0 == ey->y1);
|
||||
const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->wy0);
|
||||
const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->wy1);
|
||||
|
||||
HVX_Vector v_curr = hvx_vmem(r0);
|
||||
if (!same_row) {
|
||||
const HVX_Vector v1 = hvx_vmem(r1);
|
||||
v_curr = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, v_curr), hvx_vec_mul_f32_f32(vwy1, v1));
|
||||
}
|
||||
|
||||
HVX_Vector v_prev = Q6_V_vror_VR(v_curr, 4);
|
||||
|
||||
for (uint32_t ix = 0; ix < src_x; ix += VLEN_FP32) {
|
||||
HVX_Vector v_next_chunk;
|
||||
const uint32_t next_ix = ix + VLEN_FP32;
|
||||
if (next_ix < src_x) {
|
||||
v_next_chunk = hvx_vmem(r0 + next_ix);
|
||||
if (!same_row) {
|
||||
const HVX_Vector v1 = hvx_vmem(r1 + next_ix);
|
||||
v_next_chunk = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, v_next_chunk), hvx_vec_mul_f32_f32(vwy1, v1));
|
||||
}
|
||||
} else {
|
||||
v_next_chunk = Q6_V_vror_VR(v_curr, 124);
|
||||
}
|
||||
|
||||
const HVX_Vector v_shl = Q6_V_valign_VVR(v_curr, v_prev, 124);
|
||||
const HVX_Vector v_shr = Q6_V_valign_VVR(v_next_chunk, v_curr, 4);
|
||||
|
||||
const HVX_Vector ve = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(c_075, v_curr), hvx_vec_mul_f32_f32(c_025, v_shl));
|
||||
const HVX_Vector vo = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(c_075, v_curr), hvx_vec_mul_f32_f32(c_025, v_shr));
|
||||
|
||||
const HVX_VectorPair p = Q6_W_vshuff_VVR(vo, ve, -4);
|
||||
const HVX_Vector out0 = Q6_V_lo_W(p);
|
||||
const HVX_Vector out1 = Q6_V_hi_W(p);
|
||||
|
||||
const uint32_t ox0 = 2 * ix;
|
||||
hvx_vmem(dst_row + ox0) = out0;
|
||||
hvx_vmem(dst_row + ox0 + VLEN_FP32) = out1;
|
||||
|
||||
v_prev = v_curr;
|
||||
v_curr = v_next_chunk;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void compute_bilinear_antialias_chunk(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
) {
|
||||
(void) R_buf;
|
||||
(void) gather_tmp;
|
||||
const uint32_t src_x = ctx->octx->src[0]->ne[0];
|
||||
const uint32_t dst_x = ctx->octx->dst->ne[0];
|
||||
const struct htp_upscale_kernel_params * p = ctx->kparams;
|
||||
const struct upscale_antialias_x * lut_x = (const struct upscale_antialias_x *) ctx->lut_x;
|
||||
const struct upscale_antialias_y * lut_y = (const struct upscale_antialias_y *) ctx->lut_y;
|
||||
|
||||
const float invscale1 = p->invscale1;
|
||||
const float invscale0 = p->invscale0;
|
||||
|
||||
for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
|
||||
const struct upscale_antialias_y * ey = &lut_y[oy];
|
||||
const float yf = ey->yf;
|
||||
const int32_t y_min = ey->y_min;
|
||||
const int32_t y_max = ey->y_max;
|
||||
|
||||
float * dst_row = dstb + (oy - dst_y_start) * dst_x;
|
||||
|
||||
for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) {
|
||||
const uint32_t rem = dst_x - ox;
|
||||
const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
|
||||
float tmp[VLEN_FP32] __attribute__((aligned(128)));
|
||||
const uint32_t cnt = rem < VLEN_FP32 ? rem : VLEN_FP32;
|
||||
|
||||
for (uint32_t l = 0; l < cnt; ++l) {
|
||||
const uint32_t cur_ox = ox + l;
|
||||
const struct upscale_antialias_x * ex = &lut_x[cur_ox];
|
||||
const float xf = ex->xf;
|
||||
const int32_t x_min = ex->x_min;
|
||||
const int32_t x_max = ex->x_max;
|
||||
|
||||
float val = 0.0f;
|
||||
float total_weight = 0.0f;
|
||||
|
||||
for (int32_t sy = y_min; sy < y_max; ++sy) {
|
||||
const float diff_y = fabsf((float) sy - yf + p->pixel_offset) * invscale1;
|
||||
const float weight_y = MAX(1.0f - diff_y, 0.0f);
|
||||
if (weight_y <= 0.0f) {
|
||||
continue;
|
||||
}
|
||||
const float * r = srcb + (uint32_t) (sy - (int32_t) src_y_start) * src_x;
|
||||
|
||||
for (int32_t sx = x_min; sx < x_max; ++sx) {
|
||||
const float diff_x = fabsf((float) sx - xf + p->pixel_offset) * invscale0;
|
||||
const float weight_x = MAX(1.0f - diff_x, 0.0f);
|
||||
const float weight = weight_x * weight_y;
|
||||
if (weight <= 0.0f) {
|
||||
continue;
|
||||
}
|
||||
val += r[sx] * weight;
|
||||
total_weight += weight;
|
||||
}
|
||||
}
|
||||
|
||||
tmp[l] = (total_weight > 0.0f) ? (val / total_weight) : 0.0f;
|
||||
}
|
||||
|
||||
hvx_vec_store_u(dst_row + ox, nbytes, *(const HVX_Vector *) tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void compute_bicubic_chunk(
|
||||
const struct upscale_context * ctx,
|
||||
const float * srcb,
|
||||
float * dstb,
|
||||
float * R_buf,
|
||||
float * gather_tmp,
|
||||
uint32_t dst_y_start,
|
||||
uint32_t dst_y_count,
|
||||
uint32_t src_y_start
|
||||
) {
|
||||
(void) R_buf;
|
||||
(void) gather_tmp;
|
||||
const uint32_t src_x = ctx->octx->src[0]->ne[0];
|
||||
const uint32_t dst_x = ctx->octx->dst->ne[0];
|
||||
const struct upscale_bicubic_x * lut_x = (const struct upscale_bicubic_x *) ctx->lut_x;
|
||||
const struct upscale_bicubic_y * lut_y = (const struct upscale_bicubic_y *) ctx->lut_y;
|
||||
|
||||
for (uint32_t oy = dst_y_start; oy < dst_y_start + dst_y_count; ++oy) {
|
||||
const struct upscale_bicubic_y * ey = &lut_y[oy];
|
||||
const float * r0 = srcb + (uint32_t) (ey->y[0] - (int32_t) src_y_start) * src_x;
|
||||
const float * r1 = srcb + (uint32_t) (ey->y[1] - (int32_t) src_y_start) * src_x;
|
||||
const float * r2 = srcb + (uint32_t) (ey->y[2] - (int32_t) src_y_start) * src_x;
|
||||
const float * r3 = srcb + (uint32_t) (ey->y[3] - (int32_t) src_y_start) * src_x;
|
||||
|
||||
float * dst_row = dstb + (oy - dst_y_start) * dst_x;
|
||||
|
||||
const HVX_Vector vwy0 = hvx_vec_splat_f32(ey->w[0]);
|
||||
const HVX_Vector vwy1 = hvx_vec_splat_f32(ey->w[1]);
|
||||
const HVX_Vector vwy2 = hvx_vec_splat_f32(ey->w[2]);
|
||||
const HVX_Vector vwy3 = hvx_vec_splat_f32(ey->w[3]);
|
||||
|
||||
for (uint32_t ox = 0; ox < dst_x; ox += VLEN_FP32) {
|
||||
const uint32_t rem = dst_x - ox;
|
||||
const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
|
||||
|
||||
float v0[VLEN_FP32] __attribute__((aligned(128)));
|
||||
float v1[VLEN_FP32] __attribute__((aligned(128)));
|
||||
float v2[VLEN_FP32] __attribute__((aligned(128)));
|
||||
float v3[VLEN_FP32] __attribute__((aligned(128)));
|
||||
|
||||
const uint32_t cnt = rem < VLEN_FP32 ? rem : VLEN_FP32;
|
||||
for (uint32_t l = 0; l < cnt; ++l) {
|
||||
const struct upscale_bicubic_x * e = &lut_x[ox + l];
|
||||
v0[l] = r0[e->x[0]] * e->w[0] + r0[e->x[1]] * e->w[1] + r0[e->x[2]] * e->w[2] + r0[e->x[3]] * e->w[3];
|
||||
v1[l] = r1[e->x[0]] * e->w[0] + r1[e->x[1]] * e->w[1] + r1[e->x[2]] * e->w[2] + r1[e->x[3]] * e->w[3];
|
||||
v2[l] = r2[e->x[0]] * e->w[0] + r2[e->x[1]] * e->w[1] + r2[e->x[2]] * e->w[2] + r2[e->x[3]] * e->w[3];
|
||||
v3[l] = r3[e->x[0]] * e->w[0] + r3[e->x[1]] * e->w[1] + r3[e->x[2]] * e->w[2] + r3[e->x[3]] * e->w[3];
|
||||
}
|
||||
for (uint32_t l = cnt; l < VLEN_FP32; ++l) {
|
||||
v0[l] = v1[l] = v2[l] = v3[l] = 0.0f;
|
||||
}
|
||||
|
||||
const HVX_Vector vv0 = *(const HVX_Vector *) v0;
|
||||
const HVX_Vector vv1 = *(const HVX_Vector *) v1;
|
||||
const HVX_Vector vv2 = *(const HVX_Vector *) v2;
|
||||
const HVX_Vector vv3 = *(const HVX_Vector *) v3;
|
||||
|
||||
const HVX_Vector acc0 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy0, vv0), hvx_vec_mul_f32_f32(vwy1, vv1));
|
||||
const HVX_Vector acc1 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(vwy2, vv2), hvx_vec_mul_f32_f32(vwy3, vv3));
|
||||
const HVX_Vector val = hvx_vec_add_f32_f32(acc0, acc1);
|
||||
|
||||
hvx_vec_store_u(dst_row + ox, nbytes, val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void dma_push_src_chunk(
|
||||
dma_queue * dma_q,
|
||||
float * spad,
|
||||
const struct htp_tensor * src0,
|
||||
dma_addr_t slice_src_off,
|
||||
uint32_t src_ys,
|
||||
uint32_t src_yc
|
||||
) {
|
||||
const uint32_t src_x = src0->ne[0];
|
||||
const dma_addr_t src_addr = src0->data + slice_src_off + (dma_addr_t) src_ys * src0->nb[1];
|
||||
|
||||
if (src0->nb[0] == sizeof(float)) {
|
||||
const uint32_t row_size = src_x * sizeof(float);
|
||||
const uint32_t dst_stride = row_size;
|
||||
const uint32_t src_stride = src0->nb[1];
|
||||
dma_queue_push(dma_q, dma_make_data(spad, src_addr), dst_stride, src_stride, row_size, src_yc);
|
||||
} else {
|
||||
const uint32_t row_size = sizeof(float);
|
||||
const uint32_t dst_stride = sizeof(float);
|
||||
const uint32_t src_stride = src0->nb[0];
|
||||
const uint32_t nrows = src_x * src_yc;
|
||||
dma_queue_push(dma_q, dma_make_data(spad, src_addr), dst_stride, src_stride, row_size, nrows);
|
||||
}
|
||||
}
|
||||
|
||||
static void upscale_worker(unsigned int nth, unsigned int ith, void * data) {
|
||||
struct upscale_context * ctx = (struct upscale_context *) data;
|
||||
const struct htp_upscale_kernel_params * p = ctx->kparams;
|
||||
const struct htp_upscale_vtcm_layout * L = &ctx->layout;
|
||||
const struct htp_tensor * src0 = ctx->octx->src[0];
|
||||
const struct htp_tensor * dst = ctx->octx->dst;
|
||||
|
||||
const uint32_t first = ctx->chunk_start + ith * ctx->chunks_per_thread;
|
||||
const uint32_t last = MIN(first + ctx->chunks_per_thread, ctx->chunk_start + ctx->chunk_count);
|
||||
|
||||
if (first >= last) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith];
|
||||
dma_queue * dma_q = ctx->octx->ctx->dma[ith];
|
||||
|
||||
const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size;
|
||||
const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;
|
||||
|
||||
uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base;
|
||||
uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;
|
||||
uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;
|
||||
float * R_buf = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_r) + ith * (L->r_bytes_per_thread / sizeof(float));
|
||||
float * gather_tmp = VTCM_LAYOUT_PTR(float, vtcm_base, L->off_gather) + ith * (L->gather_bytes_per_thread / sizeof(float));
|
||||
const void * lut_y = ctx->lut_y;
|
||||
|
||||
float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };
|
||||
float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) };
|
||||
|
||||
const uint32_t dst_row_size = dst->ne[0] * sizeof(float);
|
||||
const uint32_t dst_stride = dst->nb[1];
|
||||
|
||||
const uint32_t total = last - first;
|
||||
|
||||
// Warmup: prime pipeline with initial transfers
|
||||
for (uint32_t i = 0; i < total && i < 2; ++i) {
|
||||
dma_queue_push(dma_q, dma_make_data(dst->data, dstb2[i]), dst_stride, dst_row_size, dst_row_size, 0);
|
||||
|
||||
dma_addr_t slice_src_off;
|
||||
uint32_t src_ys, src_yc;
|
||||
resolve_src_slice(src0, dst, p, lut_y, first + i, &slice_src_off, &src_ys, &src_yc);
|
||||
|
||||
dma_push_src_chunk(dma_q, srcb2[i], src0, slice_src_off, src_ys, src_yc);
|
||||
}
|
||||
|
||||
// Steady state loop
|
||||
for (uint32_t i = 0; i < total; ++i) {
|
||||
const uint32_t buf = i & 1u;
|
||||
float * srcb = srcb2[buf];
|
||||
float * dstb = dstb2[buf];
|
||||
|
||||
dma_queue_pop(dma_q);
|
||||
dma_queue_pop(dma_q);
|
||||
|
||||
dma_addr_t slice_dst_off;
|
||||
uint32_t dst_ys, dst_yc;
|
||||
resolve_dst_slice(dst, p, first + i, &slice_dst_off, &dst_ys, &dst_yc);
|
||||
|
||||
uint32_t src_ys, src_yc;
|
||||
get_src_y_window(src0, dst, p, lut_y, dst_ys, dst_yc, &src_ys, &src_yc);
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (first + i));
|
||||
ctx->compute_chunk(ctx, srcb, dstb, R_buf, gather_tmp, dst_ys, dst_yc, src_ys);
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (first + i));
|
||||
|
||||
const dma_addr_t dst_addr = dst->data + slice_dst_off + (dma_addr_t) dst_ys * dst_stride;
|
||||
dma_queue_push(dma_q, dma_make_data(dst_addr, dstb), dst_stride, dst_row_size, dst_row_size, dst_yc);
|
||||
|
||||
const uint32_t next = i + 2;
|
||||
if (next < total) {
|
||||
dma_addr_t next_src_off;
|
||||
uint32_t next_src_ys, next_src_yc;
|
||||
resolve_src_slice(src0, dst, p, lut_y, first + next, &next_src_off, &next_src_ys, &next_src_yc);
|
||||
|
||||
dma_push_src_chunk(dma_q, srcb, src0, next_src_off, next_src_ys, next_src_yc);
|
||||
}
|
||||
}
|
||||
|
||||
dma_queue_flush(dma_q);
|
||||
|
||||
(void) nth;
|
||||
}
|
||||
|
||||
int op_upscale(struct htp_ops_context * octx) {
|
||||
const struct htp_tensor * src0 = octx->src[0];
|
||||
const struct htp_tensor * dst = octx->dst;
|
||||
const struct htp_upscale_kernel_params * p =
|
||||
(const struct htp_upscale_kernel_params *) octx->kernel_params;
|
||||
|
||||
const uint32_t src_x = src0->ne[0];
|
||||
const uint32_t src_y = src0->ne[1];
|
||||
const uint32_t dst_x = dst->ne[0];
|
||||
const uint32_t dst_y = dst->ne[1];
|
||||
|
||||
uint32_t chunk_start = 0;
|
||||
uint32_t chunk_count = p->total_chunks;
|
||||
|
||||
if (octx->ctx->mdev.count > 1) {
|
||||
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
|
||||
chunk_count, htp_tensor_mdev_data_aligned(dst) ? p->chunks_per_group : 0,
|
||||
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
|
||||
chunk_start = range.start;
|
||||
chunk_count = range.count;
|
||||
}
|
||||
|
||||
if (chunk_count == 0) {
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
|
||||
const uint32_t n_threads = MIN(p->n_threads, chunk_count);
|
||||
if (!htp_ops_context_set_n_threads(octx, n_threads)) {
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
|
||||
const uint32_t chunks_per_thread = fastdiv(chunk_count + n_threads - 1, &octx->n_threads_div);
|
||||
|
||||
size_t lut_x_size = 0;
|
||||
size_t elem_y_sz = sizeof(uint32_t);
|
||||
|
||||
if (p->mode == HTP_UPSCALE_MODE_BICUBIC) {
|
||||
lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_bicubic_x), 128);
|
||||
elem_y_sz = sizeof(struct upscale_bicubic_y);
|
||||
} else if (p->antialias) {
|
||||
lut_x_size = hex_round_up(dst_x * sizeof(struct upscale_antialias_x), 128);
|
||||
elem_y_sz = sizeof(struct upscale_antialias_y);
|
||||
} else if (p->mode == HTP_UPSCALE_MODE_BILINEAR) {
|
||||
const uint32_t n_chunks = (dst_x + VLEN_FP32 - 1) / VLEN_FP32;
|
||||
lut_x_size = hex_round_up(n_chunks * sizeof(struct upscale_bilinear_x_chunk), 128);
|
||||
elem_y_sz = sizeof(struct upscale_bilinear_y);
|
||||
} else {
|
||||
lut_x_size = hex_round_up(dst_x * sizeof(uint32_t), 128);
|
||||
}
|
||||
|
||||
const size_t lut_y_size = hex_round_up(dst_y * elem_y_sz, 128);
|
||||
const size_t total_lut_size = lut_x_size + lut_y_size;
|
||||
|
||||
if (total_lut_size > octx->ctx->ddr_spad_size) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
void * base_scratch = octx->ctx->ddr_spad_base;
|
||||
void * lut_x = base_scratch;
|
||||
void * lut_y = (uint8_t *) base_scratch + lut_x_size;
|
||||
|
||||
if (p->mode == HTP_UPSCALE_MODE_NEAREST) {
|
||||
upscale_init_luts_nearest((uint32_t *) lut_x, (uint32_t *) lut_y, src0, dst, p);
|
||||
} else if (p->mode == HTP_UPSCALE_MODE_BICUBIC) {
|
||||
upscale_init_luts_bicubic((struct upscale_bicubic_x *) lut_x, (struct upscale_bicubic_y *) lut_y, src0, dst, p);
|
||||
} else if (p->antialias) {
|
||||
upscale_init_luts_antialias((struct upscale_antialias_x *) lut_x, (struct upscale_antialias_y *) lut_y, src0, dst, p);
|
||||
} else {
|
||||
upscale_init_luts_bilinear((struct upscale_bilinear_x_chunk *) lut_x, (struct upscale_bilinear_y *) lut_y, src0, dst, p);
|
||||
}
|
||||
|
||||
upscale_chunk_fn_t compute_chunk;
|
||||
switch (p->kernel_type) {
|
||||
case HTP_UPSCALE_KERNEL_NEAREST:
|
||||
compute_chunk = compute_nearest_chunk;
|
||||
break;
|
||||
case HTP_UPSCALE_KERNEL_NEAREST_2X:
|
||||
compute_chunk = compute_nearest_2x_chunk;
|
||||
break;
|
||||
case HTP_UPSCALE_KERNEL_NEAREST_4X:
|
||||
compute_chunk = compute_nearest_4x_chunk;
|
||||
break;
|
||||
case HTP_UPSCALE_KERNEL_BILINEAR:
|
||||
compute_chunk = compute_bilinear_chunk;
|
||||
break;
|
||||
case HTP_UPSCALE_KERNEL_BILINEAR_2X:
|
||||
compute_chunk = compute_bilinear_2x_chunk;
|
||||
break;
|
||||
case HTP_UPSCALE_KERNEL_BILINEAR_AA:
|
||||
compute_chunk = compute_bilinear_antialias_chunk;
|
||||
break;
|
||||
case HTP_UPSCALE_KERNEL_BICUBIC:
|
||||
compute_chunk = compute_bicubic_chunk;
|
||||
break;
|
||||
default:
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
|
||||
struct upscale_context ctx = {
|
||||
.octx = octx,
|
||||
.kparams = p,
|
||||
.compute_chunk = compute_chunk,
|
||||
.lut_x = lut_x,
|
||||
.lut_y = lut_y,
|
||||
.chunk_start = chunk_start,
|
||||
.chunk_count = chunk_count,
|
||||
.chunks_per_thread = chunks_per_thread,
|
||||
};
|
||||
|
||||
htp_upscale_vtcm_layout_build(&ctx.layout, src_x, src_y, dst_x, dst_y,
|
||||
p->dst_nrows_per_chunk, p->inv_sf1, p->filter_radius1, n_threads);
|
||||
|
||||
work_queue_run(octx->ctx->work_queue, upscale_worker, &ctx, n_threads);
|
||||
|
||||
return HTP_STATUS_OK;
|
||||
}
|
||||
@@ -1,204 +0,0 @@
|
||||
#ifndef HTP_UPSCALE_OPS_H
|
||||
#define HTP_UPSCALE_OPS_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <math.h>
|
||||
|
||||
#include "hex-common.h"
|
||||
#include "hex-fastdiv.h"
|
||||
#include "htp-vtcm.h"
|
||||
|
||||
enum htp_upscale_mode {
|
||||
HTP_UPSCALE_MODE_NEAREST = 0,
|
||||
HTP_UPSCALE_MODE_BILINEAR = 1,
|
||||
HTP_UPSCALE_MODE_BICUBIC = 2,
|
||||
};
|
||||
|
||||
enum htp_upscale_kernel_type {
|
||||
HTP_UPSCALE_KERNEL_NEAREST = 0,
|
||||
HTP_UPSCALE_KERNEL_NEAREST_2X = 1,
|
||||
HTP_UPSCALE_KERNEL_NEAREST_4X = 2,
|
||||
HTP_UPSCALE_KERNEL_BILINEAR = 3,
|
||||
HTP_UPSCALE_KERNEL_BILINEAR_2X = 4,
|
||||
HTP_UPSCALE_KERNEL_BILINEAR_AA = 5,
|
||||
HTP_UPSCALE_KERNEL_BICUBIC = 6,
|
||||
};
|
||||
|
||||
struct htp_upscale_kernel_params {
|
||||
uint8_t kernel_type;
|
||||
uint8_t mode;
|
||||
uint8_t align_corners;
|
||||
uint8_t antialias;
|
||||
|
||||
float inv_sf0;
|
||||
float inv_sf1;
|
||||
float inv_sf2;
|
||||
float inv_sf3;
|
||||
float pixel_offset;
|
||||
|
||||
float filter_radius0;
|
||||
float filter_radius1;
|
||||
float invscale0;
|
||||
float invscale1;
|
||||
|
||||
uint32_t n_threads;
|
||||
uint32_t chunks_per_group;
|
||||
|
||||
uint32_t dst_nrows_per_chunk;
|
||||
uint32_t chunks_per_slice;
|
||||
uint32_t total_chunks;
|
||||
uint32_t vtcm_size;
|
||||
|
||||
struct fastdiv_values div_chunks_per_slice;
|
||||
struct fastdiv_values div_dst_z;
|
||||
};
|
||||
|
||||
#if defined(__cplusplus)
|
||||
static_assert(sizeof(struct htp_upscale_kernel_params) <= 128, "htp_upscale_kernel_params is too large");
|
||||
#else
|
||||
_Static_assert(sizeof(struct htp_upscale_kernel_params) <= 128, "htp_upscale_kernel_params is too large");
|
||||
#endif
|
||||
|
||||
struct __attribute__((aligned(128))) upscale_bilinear_x_chunk {
|
||||
int32_t off0[32];
|
||||
int32_t off1[32];
|
||||
float w0[32];
|
||||
float dx[32];
|
||||
};
|
||||
|
||||
struct upscale_bilinear_y {
|
||||
uint32_t y0;
|
||||
uint32_t y1;
|
||||
float wy0;
|
||||
float wy1;
|
||||
};
|
||||
|
||||
struct upscale_bicubic_x {
|
||||
int32_t x[4];
|
||||
float w[4];
|
||||
};
|
||||
|
||||
struct upscale_bicubic_y {
|
||||
int32_t y[4];
|
||||
float w[4];
|
||||
};
|
||||
|
||||
struct upscale_antialias_x {
|
||||
int32_t x_min;
|
||||
int32_t x_max;
|
||||
float xf;
|
||||
};
|
||||
|
||||
struct upscale_antialias_y {
|
||||
int32_t y_min;
|
||||
int32_t y_max;
|
||||
float yf;
|
||||
};
|
||||
|
||||
struct htp_upscale_vtcm_layout {
|
||||
size_t total_bytes;
|
||||
size_t off_src;
|
||||
size_t off_dst;
|
||||
size_t off_r;
|
||||
size_t off_gather;
|
||||
size_t src_bytes_per_thread;
|
||||
size_t dst_bytes_per_thread;
|
||||
size_t r_bytes_per_thread;
|
||||
size_t gather_bytes_per_thread;
|
||||
size_t src_spad_half_size;
|
||||
size_t dst_spad_half_size;
|
||||
uint32_t max_src_chunk_rows;
|
||||
uint32_t dst_chunk_rows;
|
||||
};
|
||||
|
||||
static inline void htp_upscale_vtcm_layout_build(
|
||||
struct htp_upscale_vtcm_layout * L,
|
||||
uint32_t src_x,
|
||||
uint32_t src_y,
|
||||
uint32_t dst_x,
|
||||
uint32_t dst_y,
|
||||
uint32_t dst_chunk_rows,
|
||||
float inv_sf1,
|
||||
float filter_radius1,
|
||||
uint32_t n_threads
|
||||
) {
|
||||
uint32_t max_src_rows;
|
||||
if (dst_chunk_rows >= dst_y) {
|
||||
max_src_rows = src_y;
|
||||
} else {
|
||||
const float est = (float) dst_chunk_rows * inv_sf1 + 2.0f * filter_radius1;
|
||||
max_src_rows = (uint32_t) ceilf(est) + 4;
|
||||
if (max_src_rows > src_y) {
|
||||
max_src_rows = src_y;
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t src_chunk_bytes = src_x * max_src_rows * sizeof(float);
|
||||
const uint32_t dst_chunk_bytes = dst_x * dst_chunk_rows * sizeof(float);
|
||||
|
||||
L->max_src_chunk_rows = max_src_rows;
|
||||
L->dst_chunk_rows = dst_chunk_rows;
|
||||
L->src_spad_half_size = hex_round_up(src_chunk_bytes + 256, 128);
|
||||
L->dst_spad_half_size = hex_round_up(dst_chunk_bytes, 128);
|
||||
L->src_bytes_per_thread = 2 * L->src_spad_half_size;
|
||||
L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
|
||||
L->r_bytes_per_thread = hex_round_up(src_x * sizeof(float), 128);
|
||||
L->gather_bytes_per_thread = 256;
|
||||
|
||||
size_t off = 0;
|
||||
VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
|
||||
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
|
||||
VTCM_LAYOUT_ALLOC(off, off_r, L->r_bytes_per_thread * n_threads);
|
||||
VTCM_LAYOUT_ALLOC(off, off_gather, L->gather_bytes_per_thread * n_threads);
|
||||
L->total_bytes = off;
|
||||
}
|
||||
|
||||
static inline bool htp_upscale_solve_layout(
|
||||
struct htp_upscale_vtcm_layout * layout,
|
||||
uint32_t src_x,
|
||||
uint32_t src_y,
|
||||
uint32_t dst_x,
|
||||
uint32_t dst_y,
|
||||
float inv_sf1,
|
||||
float filter_radius1,
|
||||
uint32_t n_threads,
|
||||
size_t vtcm_budget,
|
||||
uint32_t * out_dst_chunk_rows
|
||||
) {
|
||||
// 1. Try full plane
|
||||
uint32_t chunk_rows = dst_y;
|
||||
htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
|
||||
if (layout->total_bytes <= vtcm_budget) {
|
||||
*out_dst_chunk_rows = chunk_rows;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. Try smaller chunk bands: 64, 32, 16, 8, 4, 2, 1
|
||||
chunk_rows = 64;
|
||||
while (chunk_rows > 1) {
|
||||
if (chunk_rows < dst_y) {
|
||||
htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
|
||||
if (layout->total_bytes <= vtcm_budget) {
|
||||
*out_dst_chunk_rows = chunk_rows;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
chunk_rows /= 2;
|
||||
}
|
||||
|
||||
// 3. Fallback to 1 row
|
||||
chunk_rows = 1;
|
||||
htp_upscale_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, chunk_rows, inv_sf1, filter_radius1, n_threads);
|
||||
if (layout->total_bytes <= vtcm_budget) {
|
||||
*out_dst_chunk_rows = chunk_rows;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
int op_upscale(struct htp_ops_context * octx);
|
||||
|
||||
#endif // HTP_UPSCALE_OPS_H
|
||||
@@ -512,7 +512,6 @@ static inline float ggml_e8m0_to_fp32_half(uint8_t x) {
|
||||
// UE4M3: unsigned, 4 exp bits (bias=7), 3 mantissa bits
|
||||
// Returns value * 0.5 to match kvalues_mxfp4 convention (kvalues = 2 * E2M1_float)
|
||||
static inline float ggml_ue4m3_to_fp32(uint8_t x) {
|
||||
// TODO: reject invalid unsigned E4M3 scales instead of decoding NaN as zero or ignoring the sign bit.
|
||||
if (x == 0 || x == 0x7F) {
|
||||
return 0.0f;
|
||||
}
|
||||
@@ -528,7 +527,6 @@ static inline float ggml_ue4m3_to_fp32(uint8_t x) {
|
||||
}
|
||||
|
||||
static inline uint8_t ggml_fp32_to_ue4m3(float x) {
|
||||
// TODO: use ties-to-even rounding, allow subnormal carry, and preserve finite values with exponent field 15.
|
||||
if (!(x > 0.0f)) {
|
||||
return 0;
|
||||
}
|
||||
@@ -566,62 +564,6 @@ static inline uint8_t ggml_fp32_to_ue4m3(float x) {
|
||||
return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man);
|
||||
}
|
||||
|
||||
static inline float ggml_f8_e4m3_to_fp32(uint8_t x) {
|
||||
const uint8_t ax = x & 0x7F;
|
||||
if (ax == 0x7F) {
|
||||
// OCP FP8 defines both 0x7F and 0xFF as NaN; the sign need not be preserved.
|
||||
return NAN;
|
||||
}
|
||||
|
||||
const int exp = (ax >> 3) & 0xF;
|
||||
const int man = ax & 0x7;
|
||||
// OCP FP8 rev. 1.1, section 5.1: subnormal magnitude = M * 2^-9; normal magnitude = (1 + M / 8) * 2^(E - 7).
|
||||
const float value = exp == 0
|
||||
? ldexpf((float) man, -9)
|
||||
: ldexpf(1.0f + (float) man / 8.0f, exp - 7);
|
||||
return x & 0x80 ? -value : value;
|
||||
}
|
||||
|
||||
static inline int ggml_round_to_nearest_even(float x) {
|
||||
const int value = (int) floorf(x);
|
||||
const float fraction = x - value;
|
||||
return fraction > 0.5f || (fraction == 0.5f && (value & 1)) ? value + 1 : value;
|
||||
}
|
||||
|
||||
// Supports saturating conversion only; infinities and overflow clamp to +/-448.
|
||||
static inline uint8_t ggml_fp32_to_f8_e4m3(float x) {
|
||||
const uint8_t sign = signbit(x) ? 0x80 : 0;
|
||||
x = fabsf(x);
|
||||
|
||||
if (isnan(x)) {
|
||||
// OCP FP8 leaves NaN sign handling to the implementation; use canonical 0x7F.
|
||||
return 0x7F;
|
||||
}
|
||||
if (x == 0.0f) {
|
||||
return sign;
|
||||
}
|
||||
if (isinf(x) || x >= 448.0f) {
|
||||
return sign | 0x7E;
|
||||
}
|
||||
if (x < 0.015625f) {
|
||||
// OCP FP8 rev. 1.1, section 5.1: subnormal magnitude = M * 2^(1 - bias - m) = M / 512.
|
||||
return sign | (uint8_t) ggml_round_to_nearest_even(x * 512.0f);
|
||||
}
|
||||
|
||||
int exp;
|
||||
const float mantissa = frexpf(x, &exp) * 2.0f;
|
||||
int encoded_exp = exp + 6;
|
||||
int encoded_man = ggml_round_to_nearest_even((mantissa - 1.0f) * 8.0f);
|
||||
if (encoded_man == 8) {
|
||||
encoded_man = 0;
|
||||
encoded_exp++;
|
||||
}
|
||||
if (encoded_exp > 15 || (encoded_exp == 15 && encoded_man > 6)) {
|
||||
return sign | 0x7E;
|
||||
}
|
||||
return sign | (uint8_t) (encoded_exp << 3) | (uint8_t) encoded_man;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts brain16 to float32.
|
||||
*
|
||||
|
||||
@@ -1103,6 +1103,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_meta
|
||||
{
|
||||
nsg = N_SG_MXFP4;
|
||||
nr0 = N_R0_MXFP4;
|
||||
smem = 32*sizeof(float);
|
||||
} break;
|
||||
case GGML_TYPE_Q2_K:
|
||||
{
|
||||
@@ -1398,6 +1399,7 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id(ggml_m
|
||||
{
|
||||
nsg = N_SG_MXFP4;
|
||||
nr0 = N_R0_MXFP4;
|
||||
smem = 32*sizeof(float);
|
||||
} break;
|
||||
case GGML_TYPE_Q2_K:
|
||||
{
|
||||
|
||||
@@ -362,10 +362,7 @@ bool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf);
|
||||
void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
|
||||
void ggml_metal_buffer_set_tensor (ggml_metal_buffer_t buf, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||
void ggml_metal_buffer_get_tensor (ggml_metal_buffer_t buf, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||
void ggml_metal_buffer_set_tensor_n (ggml_metal_buffer_t buf, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
|
||||
void ggml_metal_buffer_get_tensor_n (ggml_metal_buffer_t buf, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors);
|
||||
bool ggml_metal_buffer_cpy_tensor (ggml_metal_buffer_t buf, const struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||
bool ggml_metal_buffer_cpy_tensor_n (ggml_metal_buffer_t buf, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors);
|
||||
void ggml_metal_buffer_clear (ggml_metal_buffer_t buf, uint8_t value);
|
||||
|
||||
// finds the Metal buffer that contains the tensor data on the GPU device
|
||||
|
||||
@@ -1516,7 +1516,6 @@ static bool ggml_metal_supports_mul_mat_op(
|
||||
bool src0_f16_has_mv,
|
||||
bool mm_path) {
|
||||
if (!has_simdgroup_reduction ||
|
||||
op->src[0]->type == GGML_TYPE_F8_E4M3 ||
|
||||
op->src[0]->type == GGML_TYPE_NVFP4 ||
|
||||
op->src[0]->type == GGML_TYPE_TQ1_0) {
|
||||
return false;
|
||||
@@ -1934,8 +1933,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
}
|
||||
case GGML_OP_GET_ROWS:
|
||||
return op->src[0]->type != GGML_TYPE_NVFP4 &&
|
||||
op->src[0]->type != GGML_TYPE_TQ1_0 &&
|
||||
op->src[0]->type != GGML_TYPE_F8_E4M3;
|
||||
op->src[0]->type != GGML_TYPE_TQ1_0;
|
||||
case GGML_OP_SET_ROWS:
|
||||
{
|
||||
if (op->src[0]->type == GGML_TYPE_F16) {
|
||||
@@ -2438,148 +2436,6 @@ void ggml_metal_buffer_get_tensor(ggml_metal_buffer_t buf, const struct ggml_ten
|
||||
}
|
||||
}
|
||||
|
||||
static bool ggml_metal_buffer_data_span(const void * const * data, const size_t * sizes, size_t n_tensors, uintptr_t * min_data, uintptr_t * max_data) {
|
||||
bool found = false;
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uintptr_t start = (uintptr_t) data[i];
|
||||
const uintptr_t end = start + sizes[i];
|
||||
if (!found) {
|
||||
*min_data = start;
|
||||
*max_data = end;
|
||||
found = true;
|
||||
} else {
|
||||
if (start < *min_data) {
|
||||
*min_data = start;
|
||||
}
|
||||
if (end > *max_data) {
|
||||
*max_data = end;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
void ggml_metal_buffer_set_tensor_n(ggml_metal_buffer_t buf, struct ggml_tensor * const * tensors, const void * const * data,
|
||||
const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
if (buf->is_shared) {
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
memcpy((char *) tensors[i]->data + offsets[i], data[i], sizes[i]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
uintptr_t min_data = 0, max_data = 0;
|
||||
if (!ggml_metal_buffer_data_span(data, sizes, n_tensors, &min_data, &max_data)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@autoreleasepool {
|
||||
void * data_ptr = (void *) min_data;
|
||||
id<MTLBuffer> buf_src = [buf->dev->mtl_device newBufferWithBytesNoCopy:data_ptr
|
||||
length:max_data - min_data
|
||||
options:MTLResourceStorageModeShared
|
||||
deallocator:nil];
|
||||
|
||||
GGML_ASSERT(buf_src);
|
||||
|
||||
id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
|
||||
|
||||
{
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t tensor_buf_ggml = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
|
||||
ggml_metal_buffer_t tensor_buf = (ggml_metal_buffer_t) tensor_buf_ggml->context;
|
||||
struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(tensor_buf, tensors[i]);
|
||||
bid_dst.offs += offsets[i];
|
||||
|
||||
[encoder copyFromBuffer:buf_src
|
||||
sourceOffset:(uintptr_t) data[i] - min_data
|
||||
toBuffer:bid_dst.metal
|
||||
destinationOffset:bid_dst.offs
|
||||
size:sizes[i]];
|
||||
}
|
||||
|
||||
[encoder endEncoding];
|
||||
}
|
||||
|
||||
[cmd_buf commit];
|
||||
[cmd_buf waitUntilCompleted];
|
||||
|
||||
[buf_src release];
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_metal_buffer_get_tensor_n(ggml_metal_buffer_t buf, const struct ggml_tensor * const * tensors, void * const * data,
|
||||
const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
if (buf->is_shared) {
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
memcpy(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
uintptr_t min_data = 0, max_data = 0;
|
||||
if (!ggml_metal_buffer_data_span((const void * const *) data, sizes, n_tensors, &min_data, &max_data)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@autoreleasepool {
|
||||
id<MTLBuffer> buf_dst = [buf->dev->mtl_device newBufferWithBytesNoCopy:(void *) min_data
|
||||
length:max_data - min_data
|
||||
options:MTLResourceStorageModeShared
|
||||
deallocator:nil];
|
||||
|
||||
GGML_ASSERT(buf_dst);
|
||||
|
||||
id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
|
||||
|
||||
{
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t tensor_buf_ggml = tensors[i]->view_src ? tensors[i]->view_src->buffer : tensors[i]->buffer;
|
||||
ggml_metal_buffer_t tensor_buf = (ggml_metal_buffer_t) tensor_buf_ggml->context;
|
||||
struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(tensor_buf, tensors[i]);
|
||||
bid_src.offs += offsets[i];
|
||||
|
||||
[encoder copyFromBuffer:bid_src.metal
|
||||
sourceOffset:bid_src.offs
|
||||
toBuffer:buf_dst
|
||||
destinationOffset:(uintptr_t) data[i] - min_data
|
||||
size:sizes[i]];
|
||||
}
|
||||
|
||||
[encoder endEncoding];
|
||||
}
|
||||
|
||||
[cmd_buf commit];
|
||||
[cmd_buf waitUntilCompleted];
|
||||
|
||||
[buf_dst release];
|
||||
}
|
||||
}
|
||||
|
||||
bool ggml_metal_buffer_cpy_tensor(ggml_metal_buffer_t buf_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
|
||||
ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t)src->buffer->context;
|
||||
|
||||
@@ -2621,74 +2477,6 @@ bool ggml_metal_buffer_cpy_tensor(ggml_metal_buffer_t buf_dst, const struct ggml
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ggml_metal_buffer_cpy_tensor_n(ggml_metal_buffer_t buf_dst, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) {
|
||||
if (n_tensors == 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool all_shared = true;
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
|
||||
ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t) src_buf->context;
|
||||
if (!buf_dst->is_shared || !buf_src->is_shared) {
|
||||
all_shared = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (all_shared) {
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
memcpy(dsts[i]->data, srcs[i]->data, ggml_nbytes(srcs[i]));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// for private buffers, we need to use Metal blit commands
|
||||
@autoreleasepool {
|
||||
struct ggml_metal_buffer_id * bid_srcs = malloc(n_tensors * sizeof(struct ggml_metal_buffer_id));
|
||||
struct ggml_metal_buffer_id * bid_dsts = malloc(n_tensors * sizeof(struct ggml_metal_buffer_id));
|
||||
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
ggml_backend_buffer_t src_buf = srcs[i]->view_src ? srcs[i]->view_src->buffer : srcs[i]->buffer;
|
||||
ggml_backend_buffer_t dst_buf = dsts[i]->view_src ? dsts[i]->view_src->buffer : dsts[i]->buffer;
|
||||
ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t) src_buf->context;
|
||||
ggml_metal_buffer_t buf_dst_i = (ggml_metal_buffer_t) dst_buf->context;
|
||||
bid_srcs[i] = ggml_metal_buffer_get_id(buf_src, srcs[i]);
|
||||
bid_dsts[i] = ggml_metal_buffer_get_id(buf_dst_i, dsts[i]);
|
||||
|
||||
if (bid_srcs[i].metal == nil || bid_dsts[i].metal == nil) {
|
||||
free(bid_srcs);
|
||||
free(bid_dsts);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
id<MTLCommandBuffer> cmd_buf = [buf_dst->dev->mtl_queue commandBufferWithUnretainedReferences];
|
||||
|
||||
{
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
[encoder copyFromBuffer:bid_srcs[i].metal
|
||||
sourceOffset:bid_srcs[i].offs
|
||||
toBuffer:bid_dsts[i].metal
|
||||
destinationOffset:bid_dsts[i].offs
|
||||
size:ggml_nbytes(srcs[i])];
|
||||
}
|
||||
|
||||
[encoder endEncoding];
|
||||
}
|
||||
|
||||
[cmd_buf commit];
|
||||
[cmd_buf waitUntilCompleted];
|
||||
|
||||
free(bid_srcs);
|
||||
free(bid_dsts);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void ggml_metal_buffer_clear(ggml_metal_buffer_t buf, uint8_t value) {
|
||||
if (buf->is_shared) {
|
||||
memset(buf->all_data, value, buf->all_size);
|
||||
|
||||
@@ -899,9 +899,7 @@ int ggml_metal_op_unary(ggml_metal_op_t ctx, int idx) {
|
||||
if (pipeline.cnt) {
|
||||
const int n = pipeline.c4 ? ggml_nelements(op)/4 : ggml_nelements(op);
|
||||
|
||||
const int nth = MIN(256, n);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (n + nth - 1)/nth, 1, 1, nth, 1, 1);
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, 1, 1, 1);
|
||||
} else {
|
||||
const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
|
||||
const int nth = MIN(args.ne00, nth_max);
|
||||
@@ -954,9 +952,7 @@ int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) {
|
||||
|
||||
const int64_t nrows = ggml_nrows(op->src[0]);
|
||||
|
||||
// split each row over several threadgroups: with a single row (decode) one threadgroup would run on one GPU core
|
||||
const int32_t nth = std::max(1, std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), std::min(256, ne0)));
|
||||
const int32_t nk = (ne0 + nth - 1)/nth;
|
||||
const int32_t nth = std::max(1, std::min(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne00/2));
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
@@ -968,7 +964,7 @@ int ggml_metal_op_glu(ggml_metal_op_t ctx, int idx) {
|
||||
}
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, nk, nrows, 1, nth, 1, 1);
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, nrows, 1, 1, nth, 1, 1);
|
||||
|
||||
return 1;
|
||||
}
|
||||
@@ -4332,9 +4328,7 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 3);
|
||||
|
||||
if (pipeline.cnt) {
|
||||
const int nth = MIN(256, args.ne0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (args.ne0 + nth - 1)/nth, ggml_nrows(op), 1, nth, 1, 1);
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, args.ne0, ggml_nrows(op), 1, 1, 1, 1);
|
||||
} else {
|
||||
const int nth_max = MIN(256, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
|
||||
|
||||
|
||||
@@ -68,22 +68,6 @@ static void ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t bu
|
||||
ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);
|
||||
}
|
||||
|
||||
static void ggml_backend_metal_buffer_shared_set_tensor_n(ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
|
||||
|
||||
ggml_metal_buffer_set_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
static void ggml_backend_metal_buffer_shared_get_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
|
||||
|
||||
ggml_metal_buffer_get_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
@@ -96,20 +80,6 @@ static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t bu
|
||||
return ggml_metal_buffer_cpy_tensor(ctx, src, dst);
|
||||
}
|
||||
|
||||
static bool ggml_backend_metal_buffer_shared_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
|
||||
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (!ggml_backend_buffer_is_metal(srcs[i]->buffer)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return ggml_metal_buffer_cpy_tensor_n(ctx, srcs, dsts, n_tensors);
|
||||
}
|
||||
|
||||
static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
@@ -130,9 +100,6 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
|
||||
/* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_metal_buffer_shared_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ ggml_backend_metal_buffer_shared_set_tensor_n,
|
||||
/* .get_tensor_n = */ ggml_backend_metal_buffer_shared_get_tensor_n,
|
||||
/* .cpy_tensor_n = */ ggml_backend_metal_buffer_shared_cpy_tensor_n,
|
||||
};
|
||||
|
||||
// private buffer
|
||||
@@ -177,22 +144,6 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b
|
||||
ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);
|
||||
}
|
||||
|
||||
static void ggml_backend_metal_buffer_private_set_tensor_n(ggml_backend_buffer_t buffer, struct ggml_tensor * const * tensors, const void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
|
||||
|
||||
ggml_metal_buffer_set_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
static void ggml_backend_metal_buffer_private_get_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * tensors, void * const * data, const size_t * offsets, const size_t * sizes, size_t n_tensors) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
|
||||
|
||||
ggml_metal_buffer_get_tensor_n(ctx, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
@@ -205,20 +156,6 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b
|
||||
return ggml_metal_buffer_cpy_tensor(ctx, src, dst);
|
||||
}
|
||||
|
||||
static bool ggml_backend_metal_buffer_private_cpy_tensor_n(ggml_backend_buffer_t buffer, const struct ggml_tensor * const * srcs, struct ggml_tensor * const * dsts, size_t n_tensors) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
|
||||
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (!ggml_backend_buffer_is_metal(srcs[i]->buffer)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return ggml_metal_buffer_cpy_tensor_n(ctx, srcs, dsts, n_tensors);
|
||||
}
|
||||
|
||||
static void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
||||
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
|
||||
|
||||
@@ -239,9 +176,6 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {
|
||||
/* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,
|
||||
/* .clear = */ ggml_backend_metal_buffer_private_clear,
|
||||
/* .reset = */ NULL,
|
||||
/* .set_tensor_n = */ ggml_backend_metal_buffer_private_set_tensor_n,
|
||||
/* .get_tensor_n = */ ggml_backend_metal_buffer_private_get_tensor_n,
|
||||
/* .cpy_tensor_n = */ ggml_backend_metal_buffer_private_cpy_tensor_n,
|
||||
};
|
||||
|
||||
static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {
|
||||
|
||||
@@ -22,13 +22,8 @@ kernel void kernel_bin_fuse_impl(
|
||||
|
||||
if (FC_RB) {
|
||||
// row broadcast
|
||||
const uint i = tgpig.x*ntg.x + tpitg.x;
|
||||
if (i >= (uint) args.ne0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint i0 = tgpig.y*args.ne00 + i;
|
||||
const uint i1 = FC_CB ? i%args.ne10 : i;
|
||||
const uint i0 = tgpig.y*args.ne00 + tgpig.x;
|
||||
const uint i1 = FC_CB ? tgpig.x%args.ne10 : tgpig.x;
|
||||
|
||||
device const T0 * src0_row = (device const T0 *) (src0);
|
||||
device T * dst_row = (device T *) (dst);
|
||||
|
||||
@@ -3030,22 +3030,6 @@ kernel void kernel_mul_mv_iq4_xs_f32(
|
||||
kernel_mul_mv_iq4_xs_f32_impl<N_R0_IQ4_XS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
|
||||
}
|
||||
|
||||
// decode the E2M1 value in the low nibble of n, scaled by 2^-14, without a lookup table: the 3 magnitude bits
|
||||
// land in the fp16 exponent/mantissa (e=0 gives the 0.5 subnormal). same decode as MLX's fp4_e2m1
|
||||
static inline half mxfp4_dq1(uint32_t n) {
|
||||
const half c = as_type<half>(ushort((n & 7) << 9));
|
||||
return (n & 8) ? -c : c;
|
||||
}
|
||||
|
||||
// low / high nibbles of the 4 bytes of w, in byte order, scaled by 2^-14
|
||||
static inline float4 mxfp4_lo4(uint32_t w) {
|
||||
return float4(mxfp4_dq1(w), mxfp4_dq1(w >> 8), mxfp4_dq1(w >> 16), mxfp4_dq1(w >> 24));
|
||||
}
|
||||
|
||||
static inline float4 mxfp4_hi4(uint32_t w) {
|
||||
return float4(mxfp4_dq1(w >> 4), mxfp4_dq1(w >> 12), mxfp4_dq1(w >> 20), mxfp4_dq1(w >> 28));
|
||||
}
|
||||
|
||||
template<int NR0, typename args_t>
|
||||
void kernel_mul_mv_mxfp4_f32_impl(
|
||||
args_t args,
|
||||
@@ -3058,6 +3042,8 @@ void kernel_mul_mv_mxfp4_f32_impl(
|
||||
ushort sgitg) {
|
||||
const short NSG = FC_mul_mv_nsg;
|
||||
|
||||
threadgroup float * shmem_f32 = (threadgroup float *) shmem;
|
||||
|
||||
const int r0 = tgpig.x;
|
||||
const int r1 = tgpig.y;
|
||||
const int im = tgpig.z;
|
||||
@@ -3079,6 +3065,9 @@ void kernel_mul_mv_mxfp4_f32_impl(
|
||||
const short ix = tiisg/2; // 0...15
|
||||
const short it = tiisg%2; // 0 or 1
|
||||
|
||||
shmem_f32[tiisg] = kvalues_mxfp4_f[tiisg%16];
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
float4 yl[4];
|
||||
float sumf[NR0]={0.f};
|
||||
|
||||
@@ -3098,25 +3087,14 @@ void kernel_mul_mv_mxfp4_f32_impl(
|
||||
device const block_mxfp4 & xb = x[row*ns01 + ib];
|
||||
device const uint8_t * q2 = (device const uint8_t *)(xb.qs + 8*it);
|
||||
|
||||
// blocks are 17 bytes, so the 8 quant bytes are only byte aligned: read the aligned words that cover them
|
||||
// and funnel-shift. the third word is read only when needed so no word past the block end is touched
|
||||
device const uint32_t * pw = (device const uint32_t *)((ulong) q2 & ~3ul);
|
||||
const uint sh = ((uint)(ulong) q2 & 3u)*8;
|
||||
const uint32_t a = pw[0];
|
||||
const uint32_t b = pw[1];
|
||||
const uint32_t c = pw[sh ? 2 : 1];
|
||||
const uint32_t w0 = (a >> sh) | ((b << 1) << (31 - sh));
|
||||
const uint32_t w1 = (b >> sh) | ((c << 1) << (31 - sh));
|
||||
|
||||
float4 acc1 = yl[0]*mxfp4_lo4(w0);
|
||||
float4 acc2 = yl[1]*mxfp4_hi4(w0);
|
||||
float4 acc3 = yl[2]*mxfp4_lo4(w1);
|
||||
float4 acc4 = yl[3]*mxfp4_hi4(w1);
|
||||
float4 acc1 = yl[0]*float4(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]);
|
||||
float4 acc2 = yl[1]*float4(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]);
|
||||
float4 acc3 = yl[2]*float4(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]);
|
||||
float4 acc4 = yl[3]*float4(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]);
|
||||
|
||||
acc1 = (acc1 + acc3) + (acc2 + acc4);
|
||||
|
||||
// 16384 undoes the 2^-14 of the decode
|
||||
sumf[row] += (e8m0_to_fp32(xb.e)*16384.0f) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3]));
|
||||
sumf[row] += e8m0_to_fp32(xb.e) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3]));
|
||||
}
|
||||
|
||||
yb += 16 * QK_MXFP4;
|
||||
|
||||
@@ -11,12 +11,12 @@ inline constexpr T ggml_metal_unary_cast(U x) {
|
||||
#if defined(GGML_METAL_HAS_BF16)
|
||||
// MSL has no C-style cast from float vectors to bfloat vectors
|
||||
template <>
|
||||
inline bfloat4 ggml_metal_unary_cast<bfloat4, float4>(float4 x) {
|
||||
inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float4>(float4 x) {
|
||||
return bfloat4(bfloat(x[0]), bfloat(x[1]), bfloat(x[2]), bfloat(x[3]));
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bfloat4 ggml_metal_unary_cast<bfloat4, float>(float x) {
|
||||
inline constexpr bfloat4 ggml_metal_unary_cast<bfloat4, float>(float x) {
|
||||
return bfloat4(bfloat(x), bfloat(x), bfloat(x), bfloat(x));
|
||||
}
|
||||
#endif
|
||||
@@ -38,7 +38,7 @@ kernel void kernel_unary_impl(
|
||||
int i0;
|
||||
|
||||
if (FC_CNT) {
|
||||
i0 = tgpig.x*ntg.x + tpitg.x;
|
||||
i0 = tgpig.x;
|
||||
|
||||
src0_ptr = (device const T0 *) (src0);
|
||||
dst_ptr = (device T *) (dst);
|
||||
@@ -61,10 +61,6 @@ kernel void kernel_unary_impl(
|
||||
if (i0 >= args.ne0) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
if (i0 >= args.ne00*args.ne01*args.ne02*args.ne03) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const TC x = (TC) src0_ptr[i0];
|
||||
@@ -235,15 +231,14 @@ kernel void kernel_reglu(
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint2 tgpig[[threadgroup_position_in_grid]],
|
||||
uint2 tpitg[[thread_position_in_threadgroup]],
|
||||
uint2 ntg[[threads_per_threadgroup]],
|
||||
uint2 ntgs[[threadgroups_per_grid]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1);
|
||||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) {
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
const float x1 = src1_row[i0];
|
||||
|
||||
@@ -262,15 +257,14 @@ kernel void kernel_geglu(
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint2 tgpig[[threadgroup_position_in_grid]],
|
||||
uint2 tpitg[[thread_position_in_threadgroup]],
|
||||
uint2 ntg[[threads_per_threadgroup]],
|
||||
uint2 ntgs[[threadgroups_per_grid]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1);
|
||||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) {
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
const float x1 = src1_row[i0];
|
||||
|
||||
@@ -291,15 +285,14 @@ kernel void kernel_swiglu(
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint2 tgpig[[threadgroup_position_in_grid]],
|
||||
uint2 tpitg[[thread_position_in_threadgroup]],
|
||||
uint2 ntg[[threads_per_threadgroup]],
|
||||
uint2 ntgs[[threadgroups_per_grid]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1);
|
||||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) {
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
const float x1 = src1_row[i0];
|
||||
|
||||
@@ -320,15 +313,14 @@ kernel void kernel_swiglu_oai(
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint2 tgpig[[threadgroup_position_in_grid]],
|
||||
uint2 tpitg[[thread_position_in_threadgroup]],
|
||||
uint2 ntg[[threads_per_threadgroup]],
|
||||
uint2 ntgs[[threadgroups_per_grid]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1);
|
||||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) {
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
float x0 = src0_row[i0];
|
||||
float x1 = src1_row[i0];
|
||||
|
||||
@@ -353,15 +345,14 @@ kernel void kernel_swiglu_clamp(
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint2 tgpig[[threadgroup_position_in_grid]],
|
||||
uint2 tpitg[[thread_position_in_threadgroup]],
|
||||
uint2 ntg[[threads_per_threadgroup]],
|
||||
uint2 ntgs[[threadgroups_per_grid]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1);
|
||||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) {
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float gate = min((float) src0_row[i0], args.limit);
|
||||
const float up = clamp((float) src1_row[i0], -args.limit, args.limit);
|
||||
|
||||
@@ -380,15 +371,14 @@ kernel void kernel_geglu_erf(
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint2 tgpig[[threadgroup_position_in_grid]],
|
||||
uint2 tpitg[[thread_position_in_threadgroup]],
|
||||
uint2 ntg[[threads_per_threadgroup]],
|
||||
uint2 ntgs[[threadgroups_per_grid]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1);
|
||||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) {
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
const float x1 = src1_row[i0];
|
||||
|
||||
@@ -409,15 +399,14 @@ kernel void kernel_geglu_quick(
|
||||
device const char * src0,
|
||||
device const char * src1,
|
||||
device char * dst,
|
||||
uint2 tgpig[[threadgroup_position_in_grid]],
|
||||
uint2 tpitg[[thread_position_in_threadgroup]],
|
||||
uint2 ntg[[threads_per_threadgroup]],
|
||||
uint2 ntgs[[threadgroups_per_grid]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig.y*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig.y*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig.y*args.nb1);
|
||||
uint tgpig[[threadgroup_position_in_grid]],
|
||||
uint tpitg[[thread_position_in_threadgroup]],
|
||||
uint ntg[[threads_per_threadgroup]]) {
|
||||
device const T * src0_row = (device const T *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
|
||||
device const T * src1_row = (device const T *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
|
||||
device T * dst_row = (device T *) ((device char *) dst + tgpig*args.nb1);
|
||||
|
||||
for (int i0 = tgpig.x*ntg.x + tpitg.x; i0 < args.ne0; i0 += ntg.x*ntgs.x) {
|
||||
for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
|
||||
const float x0 = src0_row[i0];
|
||||
const float x1 = src1_row[i0];
|
||||
|
||||
|
||||
@@ -105,37 +105,10 @@ set(GGML_OPENCL_KERNELS
|
||||
gemv_noshuffle_q2_k_f32
|
||||
gemm_noshuffle_q2_k_f32
|
||||
gemm_noshuffle_q3_k_f32
|
||||
gemm_noshuffle_iq2_s_f32
|
||||
gemm_noshuffle_iq2_xs_f32
|
||||
gemm_noshuffle_iq2_xxs_f32
|
||||
gemm_noshuffle_iq2_xxs_q8_1_dp4a
|
||||
gemv_noshuffle_iq2_xxs_f32
|
||||
gemm_noshuffle_iq2_xs_q8_1_dp4a
|
||||
gemv_noshuffle_iq2_xs_f32
|
||||
gemm_noshuffle_iq2_s_q8_1_dp4a
|
||||
gemv_noshuffle_iq2_s_f32
|
||||
gemm_noshuffle_q2_0_f32
|
||||
gemm_noshuffle_q2_0_q8_1_dp4a
|
||||
gemv_noshuffle_q2_0_f32
|
||||
gemm_noshuffle_q3_k_q8_1_dp4a
|
||||
gemv_noshuffle_q3_k_f32
|
||||
mul_mv_q2_k_f32
|
||||
mul_mv_q3_k_f32
|
||||
iq2_s_grid
|
||||
mul_mv_iq2_s_f32
|
||||
iq2_xs_grid
|
||||
mul_mv_iq2_xs_f32
|
||||
iq2_xxs_grid
|
||||
mul_mv_iq2_xxs_f32
|
||||
mul_mv_iq2_xxs_f32_flat
|
||||
mul_mm_iq2_xxs_f32_l4_lm
|
||||
mul_mv_iq2_xs_f32_flat
|
||||
mul_mm_iq2_xs_f32_l4_lm
|
||||
mul_mv_iq2_s_f32_flat
|
||||
mul_mm_iq2_s_f32_l4_lm
|
||||
mul_mv_q2_0_f32
|
||||
mul_mv_q2_0_f32_flat
|
||||
mul_mm_q2_0_f32_l4_lm
|
||||
mul_mv_q3_k_f32_flat
|
||||
mul_mm_q3_k_f32_l4_lm
|
||||
mul_mv_q2_k_f32_flat
|
||||
@@ -233,7 +206,6 @@ set(GGML_OPENCL_KERNELS
|
||||
gemv_noshuffle_q4_k_f32
|
||||
gemv_noshuffle_q4_k_f32_o4
|
||||
gemv_noshuffle_q4_k_f32_tiled
|
||||
gemm_noshuffle_q4_K_f32_tiled_dp4a
|
||||
gemm_noshuffle_q4_k_f32
|
||||
gemv_noshuffle_q4_k_f32_32b_trans
|
||||
gemv_noshuffle_q6_k_f32
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2926,165 +2926,3 @@ kernel void kernel_restore_block_q3_k_noshuffle(
|
||||
b->hmask[ee] = (uchar)v;
|
||||
}
|
||||
}
|
||||
|
||||
struct block_iq2_s {
|
||||
half d;
|
||||
uchar qs[QK_K/4];
|
||||
uchar qh[QK_K/32];
|
||||
uchar scales[QK_K/32];
|
||||
};
|
||||
|
||||
kernel void kernel_convert_block_iq2_s(
|
||||
global struct block_iq2_s * src0,
|
||||
global uchar * dst_qs, // QK_K/8 per block
|
||||
global uchar * dst_sg, // QK_K/8
|
||||
global uchar * dst_qh, // QK_K/32
|
||||
global uchar * dst_sc, // QK_K/32
|
||||
global half * dst_d, // 1
|
||||
ulong n_blk
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blk) {
|
||||
return;
|
||||
}
|
||||
global struct block_iq2_s * b = src0 + i;
|
||||
|
||||
dst_d[i] = b->d;
|
||||
for (int j = 0; j < QK_K/8; ++j) { dst_qs[(QK_K/8) * i + j] = b->qs[j]; }
|
||||
for (int j = 0; j < QK_K/8; ++j) { dst_sg[(QK_K/8) * i + j] = b->qs[QK_K/8 + j]; }
|
||||
for (int j = 0; j < QK_K/32; ++j) { dst_qh[(QK_K/32) * i + j] = b->qh[j]; }
|
||||
for (int j = 0; j < QK_K/32; ++j) { dst_sc[(QK_K/32) * i + j] = b->scales[j]; }
|
||||
}
|
||||
|
||||
kernel void kernel_restore_block_iq2_s(
|
||||
global uchar * src_qs,
|
||||
global uchar * src_sg,
|
||||
global uchar * src_qh,
|
||||
global uchar * src_sc,
|
||||
global half * src_d,
|
||||
global struct block_iq2_s * dst,
|
||||
ulong n_blk
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blk) {
|
||||
return;
|
||||
}
|
||||
global struct block_iq2_s * b = dst + i;
|
||||
|
||||
b->d = src_d[i];
|
||||
for (int j = 0; j < QK_K/8; ++j) { b->qs[j] = src_qs[(QK_K/8) * i + j]; }
|
||||
for (int j = 0; j < QK_K/8; ++j) { b->qs[QK_K/8 + j] = src_sg[(QK_K/8) * i + j]; }
|
||||
for (int j = 0; j < QK_K/32; ++j) { b->qh[j] = src_qh[(QK_K/32) * i + j]; }
|
||||
for (int j = 0; j < QK_K/32; ++j) { b->scales[j] = src_sc[(QK_K/32) * i + j]; }
|
||||
}
|
||||
|
||||
struct block_iq2_xs {
|
||||
half d;
|
||||
ushort qs[QK_K/8];
|
||||
uchar scales[QK_K/32];
|
||||
};
|
||||
|
||||
kernel void kernel_convert_block_iq2_xs(
|
||||
global struct block_iq2_xs * src0,
|
||||
global ushort * dst_qs,
|
||||
global uchar * dst_sc,
|
||||
global half * dst_d,
|
||||
ulong n_blocks
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blocks) { return; }
|
||||
dst_d[i] = src0[i].d;
|
||||
for (int j = 0; j < 32; ++j) { dst_qs[32*i + j] = src0[i].qs[j]; }
|
||||
for (int j = 0; j < 8; ++j) { dst_sc[8*i + j] = src0[i].scales[j]; }
|
||||
}
|
||||
|
||||
kernel void kernel_restore_block_iq2_xs(
|
||||
global ushort * src_qs,
|
||||
global uchar * src_sc,
|
||||
global half * src_d,
|
||||
global struct block_iq2_xs * dst,
|
||||
ulong n_blocks
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blocks) { return; }
|
||||
dst[i].d = src_d[i];
|
||||
for (int j = 0; j < 32; ++j) { dst[i].qs[j] = src_qs[32*i + j]; }
|
||||
for (int j = 0; j < 8; ++j) { dst[i].scales[j] = src_sc[8*i + j]; }
|
||||
}
|
||||
|
||||
struct block_iq2_xxs {
|
||||
half d;
|
||||
ushort qs[QK_K/8];
|
||||
};
|
||||
|
||||
kernel void kernel_convert_block_iq2_xxs(
|
||||
global struct block_iq2_xxs * src0,
|
||||
global uchar * dst_qs,
|
||||
global uint * dst_sas,
|
||||
global half * dst_d,
|
||||
ulong n_blocks
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blocks) { return; }
|
||||
dst_d[i] = src0[i].d;
|
||||
for (int j = 0; j < 8; ++j) {
|
||||
global ushort * q = src0[i].qs + 4*j;
|
||||
uint indices = (uint)q[0] | ((uint)q[1] << 16);
|
||||
vstore4(as_uchar4(indices), 8*i + j, dst_qs);
|
||||
dst_sas[8*i + j] = (uint)q[2] | ((uint)q[3] << 16);
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_restore_block_iq2_xxs(
|
||||
global uchar * src_qs,
|
||||
global uint * src_sas,
|
||||
global half * src_d,
|
||||
global struct block_iq2_xxs * dst,
|
||||
ulong n_blocks
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blocks) { return; }
|
||||
dst[i].d = src_d[i];
|
||||
for (int j = 0; j < 8; ++j) {
|
||||
global ushort * q = dst[i].qs + 4*j;
|
||||
uint indices = as_uint(vload4(8*i + j, src_qs));
|
||||
uint sas = src_sas[8*i + j];
|
||||
q[0] = (ushort)indices;
|
||||
q[1] = (ushort)(indices >> 16);
|
||||
q[2] = (ushort)sas;
|
||||
q[3] = (ushort)(sas >> 16);
|
||||
}
|
||||
}
|
||||
|
||||
struct block_q2_0 {
|
||||
half d;
|
||||
uchar qs[16];
|
||||
};
|
||||
|
||||
kernel void kernel_convert_block_q2_0(
|
||||
global struct block_q2_0 * src0,
|
||||
global uchar * dst_qs,
|
||||
global half * dst_d,
|
||||
ulong n_blocks
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blocks) { return; }
|
||||
dst_d[i] = src0[i].d;
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
dst_qs[16*i + j] = src0[i].qs[j];
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_restore_block_q2_0(
|
||||
global uchar * src_qs,
|
||||
global half * src_d,
|
||||
global struct block_q2_0 * dst,
|
||||
ulong n_blocks
|
||||
) {
|
||||
const ulong i = get_global_id(0);
|
||||
if (i >= n_blocks) { return; }
|
||||
dst[i].d = src_d[i];
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
dst[i].qs[j] = src_qs[16*i + j];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
|
||||
#ifdef cl_qcom_reqd_sub_group_size
|
||||
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
|
||||
__attribute__((qcom_reqd_sub_group_size("full")))
|
||||
#endif
|
||||
kernel void kernel_gemm_noshuffle_iq2_s_f32(
|
||||
global const uchar * src0_qs,
|
||||
global const uchar * src0_sg,
|
||||
global const uchar * src0_qh,
|
||||
global const uchar * src0_sc,
|
||||
global const half * src0_d,
|
||||
read_only image1d_buffer_t src1,
|
||||
global float * dst,
|
||||
ulong offsetd,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int n_no_padding
|
||||
) {
|
||||
dst = (global float *)((global char *)dst + offsetd);
|
||||
const int gy = get_global_id(0);
|
||||
const int row = get_global_id(1) * 4;
|
||||
const int n_4 = n / 4;
|
||||
if (row >= m) {
|
||||
return;
|
||||
}
|
||||
|
||||
half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
|
||||
for (int i = 0; i < k; i += 16) {
|
||||
const uchar4 sc = vload4(0, src0_sc + row + (i / 32) * m);
|
||||
const half4 d = vload4(0, src0_d + row + (i / 256) * m);
|
||||
const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4((sc >> (uchar4)(4*((i / 16) & 1))) & (uchar4)(15)));
|
||||
const half4 scale = convert_half4(scale_f);
|
||||
|
||||
for (int l = 0; l < 16; l += 4) {
|
||||
const int ki = i + l;
|
||||
const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m));
|
||||
const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (ki / 32) * m));
|
||||
const uchar4 sg = vload4(0, src0_sg + row + (ki / 8) * m);
|
||||
const uint4 gi = qs | (((qh >> (uint4)(2*((ki / 8) & 3))) & (uint4)(3)) << (uint4)(8));
|
||||
const uint half_grid = (ki & 7) / 4;
|
||||
const uint4 grid = (uint4)(iq2_s_grid[2*gi.s0 + half_grid], iq2_s_grid[2*gi.s1 + half_grid],
|
||||
iq2_s_grid[2*gi.s2 + half_grid], iq2_s_grid[2*gi.s3 + half_grid]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
half8 b;
|
||||
b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
|
||||
b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
|
||||
const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
|
||||
const int4 signs = convert_int4((sg >> (uchar4)((ki & 7) + j)) & (uchar4)(1));
|
||||
const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
|
||||
c0 += b * w.s0;
|
||||
c1 += b * w.s1;
|
||||
c2 += b * w.s2;
|
||||
c3 += b * w.s3;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int idx = gy * 8 * m + row;
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define QK_K 256
|
||||
|
||||
#ifndef TILESIZE_N
|
||||
#define TILESIZE_N 32
|
||||
#endif
|
||||
|
||||
inline uint iq2_s_pack(uint gv, uint sg, uint base) {
|
||||
int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
|
||||
int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
|
||||
int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
|
||||
int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
|
||||
return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8)
|
||||
| (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
|
||||
}
|
||||
|
||||
// Pass staged operands by value; local dp4a operands miscompile on X2.
|
||||
inline int dot4_q8a_v(uint4 qw, uint4 a) {
|
||||
int r = 0;
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
|
||||
return r;
|
||||
}
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_gemm_noshuffle_iq2_s_q8_1_dp4a(
|
||||
__global const uchar * src0_qs,
|
||||
__global const uchar * src0_sg,
|
||||
__global const uchar * src0_qh,
|
||||
__global const uchar * src0_sc,
|
||||
__global const half * src0_d,
|
||||
__global const uint * src1_qa,
|
||||
__global const half * src1_da,
|
||||
__global float * dst,
|
||||
ulong offsetd,
|
||||
int m,
|
||||
int n_no_padding,
|
||||
int k
|
||||
) {
|
||||
dst = (global float *)((global char *)dst + offsetd);
|
||||
|
||||
const uint lid = get_local_id(0);
|
||||
const uint block_id_m = get_global_id(1);
|
||||
const uint block_id_n = get_global_id(2);
|
||||
|
||||
const uint row = block_id_m * 64 + lid;
|
||||
const uint col_base = block_id_n * TILESIZE_N;
|
||||
const bool row_valid = row < (uint)m;
|
||||
const uint rrow = row_valid ? row : 0;
|
||||
|
||||
const uint k_u = (uint)k >> 2;
|
||||
const uint k_b = (uint)k >> 5;
|
||||
|
||||
__local uint4 sh_qa4[TILESIZE_N][2];
|
||||
__local half sh_d[TILESIZE_N];
|
||||
|
||||
#define IQ2_S_GRID(i) iq2_s_grid[(i)]
|
||||
|
||||
#define NGROUPS (TILESIZE_N / 4)
|
||||
float4 acc[NGROUPS];
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
|
||||
|
||||
for (uint step = 0; step < (uint)k; step += 32) {
|
||||
const uint sub = step >> 5;
|
||||
const uint ib = sub >> 3;
|
||||
|
||||
const uint qhv = (uint)src0_qh[rrow + sub * (uint)m];
|
||||
const uint scv = (uint)src0_sc[rrow + sub * (uint)m];
|
||||
const float dv = (float)src0_d[rrow + ib * (uint)m];
|
||||
const float dl0 = dv * (0.5f + (float)( scv & 0xFu)) * 0.25f;
|
||||
const float dl1 = dv * (0.5f + (float)( scv >> 4) ) * 0.25f;
|
||||
|
||||
const uint gb = rrow + (step >> 3) * (uint)m;
|
||||
|
||||
uint4 qlo, qhi;
|
||||
{
|
||||
const uint gi0 = (uint)src0_qs[gb + 0u * (uint)m] | ((( qhv & 3u) << 8));
|
||||
const uint gi1 = (uint)src0_qs[gb + 1u * (uint)m] | ((((qhv >> 2) & 3u) << 8));
|
||||
const uint gi2 = (uint)src0_qs[gb + 2u * (uint)m] | ((((qhv >> 4) & 3u) << 8));
|
||||
const uint gi3 = (uint)src0_qs[gb + 3u * (uint)m] | ((((qhv >> 6) & 3u) << 8));
|
||||
const uint s0 = (uint)src0_sg[gb + 0u * (uint)m];
|
||||
const uint s1 = (uint)src0_sg[gb + 1u * (uint)m];
|
||||
const uint s2 = (uint)src0_sg[gb + 2u * (uint)m];
|
||||
const uint s3 = (uint)src0_sg[gb + 3u * (uint)m];
|
||||
qlo.s0 = iq2_s_pack(IQ2_S_GRID(2u*gi0 + 0u), s0, 0u);
|
||||
qlo.s1 = iq2_s_pack(IQ2_S_GRID(2u*gi0 + 1u), s0, 4u);
|
||||
qlo.s2 = iq2_s_pack(IQ2_S_GRID(2u*gi1 + 0u), s1, 0u);
|
||||
qlo.s3 = iq2_s_pack(IQ2_S_GRID(2u*gi1 + 1u), s1, 4u);
|
||||
qhi.s0 = iq2_s_pack(IQ2_S_GRID(2u*gi2 + 0u), s2, 0u);
|
||||
qhi.s1 = iq2_s_pack(IQ2_S_GRID(2u*gi2 + 1u), s2, 4u);
|
||||
qhi.s2 = iq2_s_pack(IQ2_S_GRID(2u*gi3 + 0u), s3, 0u);
|
||||
qhi.s3 = iq2_s_pack(IQ2_S_GRID(2u*gi3 + 1u), s3, 4u);
|
||||
}
|
||||
|
||||
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
|
||||
const uint t = idx >> 1;
|
||||
const uint v = idx & 1;
|
||||
const uint c = col_base + t;
|
||||
sh_qa4[t][v] = (c < (uint)n_no_padding)
|
||||
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
|
||||
: (uint4)(0u);
|
||||
}
|
||||
if (lid < TILESIZE_N) {
|
||||
const uint c = col_base + lid;
|
||||
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
|
||||
#define IQ2_S_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
|
||||
+ dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])))
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) {
|
||||
const int b = g * 4;
|
||||
float4 rf;
|
||||
rf.s0 = IQ2_S_COL(b+0); rf.s1 = IQ2_S_COL(b+1);
|
||||
rf.s2 = IQ2_S_COL(b+2); rf.s3 = IQ2_S_COL(b+3);
|
||||
acc[g] += LD4(sh_d, b) * rf;
|
||||
}
|
||||
#undef IQ2_S_COL
|
||||
#undef LD4
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
}
|
||||
|
||||
if (!row_valid) {
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) {
|
||||
const uint b = (uint)(g * 4);
|
||||
const float4 a = acc[g];
|
||||
const uint c0 = col_base + b;
|
||||
if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
|
||||
if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
|
||||
if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
|
||||
if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
|
||||
}
|
||||
#undef NGROUPS
|
||||
#undef IQ2_S_GRID
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
|
||||
#ifdef cl_qcom_reqd_sub_group_size
|
||||
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
|
||||
__attribute__((qcom_reqd_sub_group_size("full")))
|
||||
#endif
|
||||
kernel void kernel_gemm_noshuffle_iq2_xs_f32(
|
||||
global const ushort * src0_qs,
|
||||
global const uchar * src0_sc,
|
||||
global const half * src0_d,
|
||||
read_only image1d_buffer_t src1,
|
||||
global float * dst,
|
||||
ulong offsetd,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int n_no_padding
|
||||
) {
|
||||
dst = (global float *)((global char *)dst + offsetd);
|
||||
const int gy = get_global_id(0);
|
||||
const int row = get_global_id(1) * 4;
|
||||
const int n_4 = n / 4;
|
||||
if (row >= m) {
|
||||
return;
|
||||
}
|
||||
|
||||
half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
|
||||
for (int i = 0; i < k; i += 16) {
|
||||
const uchar4 sc = vload4(0, src0_sc + row + (i / 32) * m);
|
||||
const half4 d = vload4(0, src0_d + row + (i / 256) * m);
|
||||
const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4((sc >> (uchar4)(4*((i / 16) & 1))) & (uchar4)(15)));
|
||||
const half4 scale = convert_half4(scale_f);
|
||||
|
||||
for (int l = 0; l < 16; l += 4) {
|
||||
const int ki = i + l;
|
||||
const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m));
|
||||
const uint4 code = qs >> (uint4)(9);
|
||||
const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7));
|
||||
const uint4 gi = qs & (uint4)(511);
|
||||
const uint half_grid = (ki & 7) / 4;
|
||||
const uint4 grid = (uint4)(iq2_xs_grid[2*gi.s0 + half_grid], iq2_xs_grid[2*gi.s1 + half_grid],
|
||||
iq2_xs_grid[2*gi.s2 + half_grid], iq2_xs_grid[2*gi.s3 + half_grid]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
half8 b;
|
||||
b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
|
||||
b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
|
||||
const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
|
||||
const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1));
|
||||
const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
|
||||
c0 += b * w.s0;
|
||||
c1 += b * w.s1;
|
||||
c2 += b * w.s2;
|
||||
c3 += b * w.s3;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int idx = gy * 8 * m + row;
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define QK_K 256
|
||||
|
||||
#ifndef TILESIZE_N
|
||||
#define TILESIZE_N 32
|
||||
#endif
|
||||
|
||||
inline uint iq2_xs_pack(uint gv, uint sg, uint base) {
|
||||
int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
|
||||
int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
|
||||
int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
|
||||
int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
|
||||
return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8)
|
||||
| (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
|
||||
}
|
||||
|
||||
// Pass staged operands by value; local dp4a operands miscompile on X2.
|
||||
inline int dot4_q8a_v(uint4 qw, uint4 a) {
|
||||
int r = 0;
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
|
||||
return r;
|
||||
}
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a(
|
||||
__global const ushort * src0_qs,
|
||||
__global const uchar * src0_sc,
|
||||
__global const half * src0_d,
|
||||
__global const uint * src1_qa,
|
||||
__global const half * src1_da,
|
||||
__global float * dst,
|
||||
ulong offsetd,
|
||||
int m,
|
||||
int n_no_padding,
|
||||
int k
|
||||
) {
|
||||
dst = (global float *)((global char *)dst + offsetd);
|
||||
|
||||
const uint lid = get_local_id(0);
|
||||
const uint block_id_m = get_global_id(1);
|
||||
const uint block_id_n = get_global_id(2);
|
||||
|
||||
const uint row = block_id_m * 64 + lid;
|
||||
const uint col_base = block_id_n * TILESIZE_N;
|
||||
const bool row_valid = row < (uint)m;
|
||||
const uint rrow = row_valid ? row : 0;
|
||||
|
||||
const uint k_u = (uint)k >> 2;
|
||||
const uint k_b = (uint)k >> 5;
|
||||
|
||||
__local uint4 sh_qa4[TILESIZE_N][2];
|
||||
__local half sh_d[TILESIZE_N];
|
||||
|
||||
#define IQ2_XS_GRID(i) iq2_xs_grid[(i)]
|
||||
|
||||
#define NGROUPS (TILESIZE_N / 4)
|
||||
float4 acc[NGROUPS];
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
|
||||
|
||||
for (uint step = 0; step < (uint)k; step += 32) {
|
||||
const uint sub = step >> 5;
|
||||
const uint ib = sub >> 3;
|
||||
|
||||
const uint scv = (uint)src0_sc[rrow + sub * (uint)m];
|
||||
const float dv = (float)src0_d[rrow + ib * (uint)m];
|
||||
const float dl0 = dv * (0.5f + (float)( scv & 0xFu)) * 0.25f;
|
||||
const float dl1 = dv * (0.5f + (float)( scv >> 4) ) * 0.25f;
|
||||
|
||||
const uint gb = rrow + (step >> 3) * (uint)m;
|
||||
|
||||
uint4 qlo, qhi;
|
||||
{
|
||||
const uint q0 = (uint)src0_qs[gb + 0u * (uint)m];
|
||||
const uint q1 = (uint)src0_qs[gb + 1u * (uint)m];
|
||||
const uint q2 = (uint)src0_qs[gb + 2u * (uint)m];
|
||||
const uint q3 = (uint)src0_qs[gb + 3u * (uint)m];
|
||||
const uint gi0 = (q0 & 511u) << 1;
|
||||
const uint gi1 = (q1 & 511u) << 1;
|
||||
const uint gi2 = (q2 & 511u) << 1;
|
||||
const uint gi3 = (q3 & 511u) << 1;
|
||||
const uint s0 = iq2_xs_signs(q0 >> 9);
|
||||
const uint s1 = iq2_xs_signs(q1 >> 9);
|
||||
const uint s2 = iq2_xs_signs(q2 >> 9);
|
||||
const uint s3 = iq2_xs_signs(q3 >> 9);
|
||||
qlo.s0 = iq2_xs_pack(IQ2_XS_GRID(gi0 + 0u), s0, 0u);
|
||||
qlo.s1 = iq2_xs_pack(IQ2_XS_GRID(gi0 + 1u), s0, 4u);
|
||||
qlo.s2 = iq2_xs_pack(IQ2_XS_GRID(gi1 + 0u), s1, 0u);
|
||||
qlo.s3 = iq2_xs_pack(IQ2_XS_GRID(gi1 + 1u), s1, 4u);
|
||||
qhi.s0 = iq2_xs_pack(IQ2_XS_GRID(gi2 + 0u), s2, 0u);
|
||||
qhi.s1 = iq2_xs_pack(IQ2_XS_GRID(gi2 + 1u), s2, 4u);
|
||||
qhi.s2 = iq2_xs_pack(IQ2_XS_GRID(gi3 + 0u), s3, 0u);
|
||||
qhi.s3 = iq2_xs_pack(IQ2_XS_GRID(gi3 + 1u), s3, 4u);
|
||||
}
|
||||
|
||||
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
|
||||
const uint t = idx >> 1;
|
||||
const uint v = idx & 1;
|
||||
const uint c = col_base + t;
|
||||
sh_qa4[t][v] = (c < (uint)n_no_padding)
|
||||
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
|
||||
: (uint4)(0u);
|
||||
}
|
||||
if (lid < TILESIZE_N) {
|
||||
const uint c = col_base + lid;
|
||||
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
|
||||
#define IQ2_XS_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
|
||||
+ dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])))
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) {
|
||||
const int b = g * 4;
|
||||
float4 rf;
|
||||
rf.s0 = IQ2_XS_COL(b+0); rf.s1 = IQ2_XS_COL(b+1);
|
||||
rf.s2 = IQ2_XS_COL(b+2); rf.s3 = IQ2_XS_COL(b+3);
|
||||
acc[g] += LD4(sh_d, b) * rf;
|
||||
}
|
||||
#undef IQ2_XS_COL
|
||||
#undef LD4
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
}
|
||||
|
||||
if (!row_valid) {
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) {
|
||||
const uint b = (uint)(g * 4);
|
||||
const float4 a = acc[g];
|
||||
const uint c0 = col_base + b;
|
||||
if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
|
||||
if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
|
||||
if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
|
||||
if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
|
||||
}
|
||||
#undef NGROUPS
|
||||
#undef IQ2_XS_GRID
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
|
||||
#ifdef cl_qcom_reqd_sub_group_size
|
||||
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
|
||||
__attribute__((qcom_reqd_sub_group_size("full")))
|
||||
#endif
|
||||
kernel void kernel_gemm_noshuffle_iq2_xxs_f32(
|
||||
global const uchar * src0_qs,
|
||||
global const uint * src0_sas,
|
||||
global const half * src0_d,
|
||||
read_only image1d_buffer_t src1,
|
||||
global float * dst,
|
||||
ulong offsetd,
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int n_no_padding
|
||||
) {
|
||||
dst = (global float *)((global char *)dst + offsetd);
|
||||
const int gy = get_global_id(0);
|
||||
const int row = get_global_id(1) * 4;
|
||||
const int n_4 = n / 4;
|
||||
if (row >= m) {
|
||||
return;
|
||||
}
|
||||
|
||||
half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
|
||||
for (int i = 0; i < k; i += 16) {
|
||||
const uint4 sas = vload4(0, src0_sas + row + (i / 32) * m);
|
||||
const half4 d = vload4(0, src0_d + row + (i / 256) * m);
|
||||
const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4(sas >> (uint4)(28)));
|
||||
const half4 scale = convert_half4(scale_f);
|
||||
|
||||
for (int l = 0; l < 16; l += 4) {
|
||||
const int ki = i + l;
|
||||
const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m));
|
||||
const uint4 code = (sas >> (uint4)(7*((ki / 8) & 3))) & (uint4)(127);
|
||||
const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7));
|
||||
const uint half_grid = (ki & 7) / 4;
|
||||
const uint4 grid = (uint4)(iq2_xxs_grid[2*gi.s0 + half_grid], iq2_xxs_grid[2*gi.s1 + half_grid],
|
||||
iq2_xxs_grid[2*gi.s2 + half_grid], iq2_xxs_grid[2*gi.s3 + half_grid]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
half8 b;
|
||||
b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
|
||||
b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
|
||||
const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
|
||||
const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1));
|
||||
const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
|
||||
c0 += b * w.s0;
|
||||
c1 += b * w.s1;
|
||||
c2 += b * w.s2;
|
||||
c3 += b * w.s3;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int idx = gy * 8 * m + row;
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
|
||||
idx += m;
|
||||
}
|
||||
if (idx + 3 < m * n_no_padding) {
|
||||
vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define QK_K 256
|
||||
|
||||
#ifndef TILESIZE_N
|
||||
#define TILESIZE_N 32
|
||||
#endif
|
||||
|
||||
inline uint iq2_xxs_pack(uint gv, uint sg, uint base) {
|
||||
int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
|
||||
int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
|
||||
int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
|
||||
int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
|
||||
return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8)
|
||||
| (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
|
||||
}
|
||||
|
||||
// Pass staged operands by value; local dp4a operands miscompile on X2.
|
||||
inline int dot4_q8a_v(uint4 qw, uint4 a) {
|
||||
int r = 0;
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
|
||||
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
|
||||
return r;
|
||||
}
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a(
|
||||
__global const uchar * src0_qs,
|
||||
__global const uint * src0_sas,
|
||||
__global const half * src0_d,
|
||||
__global const uint * src1_qa,
|
||||
__global const half * src1_da,
|
||||
__global float * dst,
|
||||
ulong offsetd,
|
||||
int m,
|
||||
int n_no_padding,
|
||||
int k
|
||||
) {
|
||||
dst = (global float *)((global char *)dst + offsetd);
|
||||
|
||||
const uint lid = get_local_id(0);
|
||||
const uint block_id_m = get_global_id(1);
|
||||
const uint block_id_n = get_global_id(2);
|
||||
|
||||
const uint row = block_id_m * 64 + lid;
|
||||
const uint col_base = block_id_n * TILESIZE_N;
|
||||
const bool row_valid = row < (uint)m;
|
||||
const uint rrow = row_valid ? row : 0;
|
||||
|
||||
const uint k_u = (uint)k >> 2;
|
||||
const uint k_b = (uint)k >> 5;
|
||||
|
||||
__local uint4 sh_qa4[TILESIZE_N][2];
|
||||
__local half sh_d[TILESIZE_N];
|
||||
|
||||
#define IQ2_XXS_GRID(i) iq2_xxs_grid[(i)]
|
||||
|
||||
#define NGROUPS (TILESIZE_N / 4)
|
||||
float4 acc[NGROUPS];
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
|
||||
|
||||
for (uint step = 0; step < (uint)k; step += 32) {
|
||||
const uint sub = step >> 5;
|
||||
const uint ib = sub >> 3;
|
||||
|
||||
const uint aux = src0_sas[rrow + sub * (uint)m];
|
||||
const float dv = (float)src0_d[rrow + ib * (uint)m];
|
||||
const float dl = dv * (0.5f + (float)(aux >> 28)) * 0.25f;
|
||||
|
||||
const uint gb = rrow + (step >> 3) * (uint)m;
|
||||
|
||||
uint4 qlo, qhi;
|
||||
{
|
||||
const uint gi0 = (uint)src0_qs[gb + 0u * (uint)m] << 1;
|
||||
const uint gi1 = (uint)src0_qs[gb + 1u * (uint)m] << 1;
|
||||
const uint gi2 = (uint)src0_qs[gb + 2u * (uint)m] << 1;
|
||||
const uint gi3 = (uint)src0_qs[gb + 3u * (uint)m] << 1;
|
||||
const uint s0 = iq2_xxs_signs((aux >> 0) & 127u);
|
||||
const uint s1 = iq2_xxs_signs((aux >> 7) & 127u);
|
||||
const uint s2 = iq2_xxs_signs((aux >> 14) & 127u);
|
||||
const uint s3 = iq2_xxs_signs((aux >> 21) & 127u);
|
||||
qlo.s0 = iq2_xxs_pack(IQ2_XXS_GRID(gi0 + 0u), s0, 0u);
|
||||
qlo.s1 = iq2_xxs_pack(IQ2_XXS_GRID(gi0 + 1u), s0, 4u);
|
||||
qlo.s2 = iq2_xxs_pack(IQ2_XXS_GRID(gi1 + 0u), s1, 0u);
|
||||
qlo.s3 = iq2_xxs_pack(IQ2_XXS_GRID(gi1 + 1u), s1, 4u);
|
||||
qhi.s0 = iq2_xxs_pack(IQ2_XXS_GRID(gi2 + 0u), s2, 0u);
|
||||
qhi.s1 = iq2_xxs_pack(IQ2_XXS_GRID(gi2 + 1u), s2, 4u);
|
||||
qhi.s2 = iq2_xxs_pack(IQ2_XXS_GRID(gi3 + 0u), s3, 0u);
|
||||
qhi.s3 = iq2_xxs_pack(IQ2_XXS_GRID(gi3 + 1u), s3, 4u);
|
||||
}
|
||||
|
||||
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
|
||||
const uint t = idx >> 1;
|
||||
const uint v = idx & 1;
|
||||
const uint c = col_base + t;
|
||||
sh_qa4[t][v] = (c < (uint)n_no_padding)
|
||||
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
|
||||
: (uint4)(0u);
|
||||
}
|
||||
if (lid < TILESIZE_N) {
|
||||
const uint c = col_base + lid;
|
||||
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
|
||||
#define IQ2_XXS_COL(b) (dl * (float)(dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
|
||||
+ dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1]))))
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) {
|
||||
const int b = g * 4;
|
||||
float4 rf;
|
||||
rf.s0 = IQ2_XXS_COL(b+0); rf.s1 = IQ2_XXS_COL(b+1);
|
||||
rf.s2 = IQ2_XXS_COL(b+2); rf.s3 = IQ2_XXS_COL(b+3);
|
||||
acc[g] += LD4(sh_d, b) * rf;
|
||||
}
|
||||
#undef IQ2_XXS_COL
|
||||
#undef LD4
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
}
|
||||
|
||||
if (!row_valid) {
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int g = 0; g < NGROUPS; ++g) {
|
||||
const uint b = (uint)(g * 4);
|
||||
const float4 a = acc[g];
|
||||
const uint c0 = col_base + b;
|
||||
if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
|
||||
if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
|
||||
if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
|
||||
if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
|
||||
}
|
||||
#undef NGROUPS
|
||||
#undef IQ2_XXS_GRID
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user